diff --git a/.github/workflows/test-migrations.yml b/.github/workflows/test-migrations.yml new file mode 100644 index 000000000..1def94ff7 --- /dev/null +++ b/.github/workflows/test-migrations.yml @@ -0,0 +1,85 @@ +name: Migrations + +on: + workflow_dispatch: + pull_request: + paths: + # Migration implementation files + - 'repo/fsrepo/migrations/**' + - 'test/cli/migrations/**' + # Config and repo handling + - 'repo/fsrepo/**' + # This workflow file itself + - '.github/workflows/test-migrations.yml' + push: + branches: + - 'master' + - 'release-*' + paths: + - 'repo/fsrepo/migrations/**' + - 'test/cli/migrations/**' + - 'repo/fsrepo/**' + - '.github/workflows/test-migrations.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + TEST_VERBOSE: 1 + IPFS_CHECK_RCMGR_DEFAULTS: 1 + defaults: + run: + shell: bash + steps: + - name: Check out Kubo + uses: actions/checkout@v5 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build kubo binary + run: | + make build + echo "Built ipfs binary at $(pwd)/cmd/ipfs/" + + - name: Add kubo to PATH + run: | + echo "$(pwd)/cmd/ipfs" >> $GITHUB_PATH + + - name: Verify ipfs in PATH + run: | + which ipfs || echo "ipfs not in PATH" + ipfs version || echo "Failed to run ipfs version" + + - name: Run migration unit tests + run: | + go test ./repo/fsrepo/migrations/... + + - name: Run CLI migration tests + env: + IPFS_PATH: ${{ runner.temp }}/ipfs-test + run: | + export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" + which ipfs || echo "ipfs not found in PATH" + ipfs version || echo "Failed to run ipfs version" + go test ./test/cli/migrations/... + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-test-results + path: | + test/**/*.log + ${{ runner.temp }}/ipfs-test/ diff --git a/bin/check_go_version b/bin/check_go_version deleted file mode 100755 index 74320010b..000000000 --- a/bin/check_go_version +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Check that the go version is at least equal to a minimum version -# number. -# -# Call it for example like this: -# -# $ check_go_version "1.5.2" -# - -USAGE="$0 GO_MIN_VERSION" - -die() { - printf >&2 "fatal: %s\n" "$@" - exit 1 -} - -# Get arguments - -test "$#" -eq "1" || die "This program must be passed exactly 1 arguments" "Usage: $USAGE" - -GO_MIN_VERSION="$1" - -UPGRADE_MSG="Please take a look at https://golang.org/doc/install to install or upgrade go." - -# Get path to the directory containing this file -# If $0 has no slashes, uses "./" -PREFIX=$(expr "$0" : "\(.*\/\)") || PREFIX='./' -# Include the 'check_at_least_version' function -. ${PREFIX}check_version - -# Check that the go binary exists and is in the path - -GOCC=${GOCC="go"} - -type ${GOCC} >/dev/null 2>&1 || die_upgrade "go is not installed or not in the PATH!" - -# Check the go binary version - -VERS_STR=$(${GOCC} version 2>&1) || die "'go version' failed with output: $VERS_STR" - -GO_CUR_VERSION=$(expr "$VERS_STR" : ".*go version.* go\([^[:space:]]*\) .*") || die "Invalid 'go version' output: $VERS_STR" - -check_at_least_version "$GO_MIN_VERSION" "$GO_CUR_VERSION" "${GOCC}" diff --git a/bin/check_version b/bin/check_version deleted file mode 100755 index 25007002c..000000000 --- a/bin/check_version +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh - -if test "x$UPGRADE_MSG" = "x"; then - printf >&2 "fatal: Please set '"'$UPGRADE_MSG'"' before sourcing this script\n" - exit 1 -fi - -die_upgrade() { - printf >&2 "fatal: %s\n" "$@" - printf >&2 "=> %s\n" "$UPGRADE_MSG" - exit 1 -} - -major_number() { - vers="$1" - - # Hack around 'expr' exiting with code 1 when it outputs 0 - case "$vers" in - 0) echo "0" ;; - 0.*) echo "0" ;; - *) expr "$vers" : "\([^.]*\).*" || return 1 - esac -} - -check_at_least_version() { - MIN_VERS="$1" - CUR_VERS="$2" - PROG_NAME="$3" - - # Get major, minor and fix numbers for each version - MIN_MAJ=$(major_number "$MIN_VERS") || die "No major version number in '$MIN_VERS' for '$PROG_NAME'" - CUR_MAJ=$(major_number "$CUR_VERS") || die "No major version number in '$CUR_VERS' for '$PROG_NAME'" - - # We expect a version to be of form X.X.X - # if the second dot doesn't match, we consider it a prerelease - - if MIN_MIN=$(expr "$MIN_VERS" : "[^.]*\.\([0-9][0-9]*\)"); then - # this captured digit is necessary, since expr returns code 1 if the output is empty - if expr "$MIN_VERS" : "[^.]*\.[0-9]*\([0-9]\.\|[0-9]\$\)" >/dev/null; then - MIN_PRERELEASE="0" - else - MIN_PRERELEASE="1" - fi - MIN_FIX=$(expr "$MIN_VERS" : "[^.]*\.[0-9][0-9]*[^0-9][^0-9]*\([0-9][0-9]*\)") || MIN_FIX="0" - else - MIN_MIN="0" - MIN_PRERELEASE="0" - MIN_FIX="0" - fi - if CUR_MIN=$(expr "$CUR_VERS" : "[^.]*\.\([0-9][0-9]*\)"); then - # this captured digit is necessary, since expr returns code 1 if the output is empty - if expr "$CUR_VERS" : "[^.]*\.[0-9]*\([0-9]\.\|[0-9]\$\)" >/dev/null; then - CUR_PRERELEASE="0" - else - CUR_PRERELEASE="1" - fi - CUR_FIX=$(expr "$CUR_VERS" : "[^.]*\.[0-9][0-9]*[^0-9][^0-9]*\([0-9][0-9]*\)") || CUR_FIX="0" - else - CUR_MIN="0" - CUR_PRERELEASE="0" - CUR_FIX="0" - fi - - # Compare versions - VERS_LEAST="$PROG_NAME version '$CUR_VERS' should be at least '$MIN_VERS'" - test "$CUR_MAJ" -lt "$MIN_MAJ" && die_upgrade "$VERS_LEAST" - test "$CUR_MAJ" -gt "$MIN_MAJ" || { - test "$CUR_MIN" -lt "$MIN_MIN" && die_upgrade "$VERS_LEAST" - test "$CUR_MIN" -gt "$MIN_MIN" || { - test "$CUR_PRERELEASE" -gt "$MIN_PRERELEASE" && die_upgrade "$VERS_LEAST" - test "$CUR_PRERELEASE" -lt "$MIN_PRERELEASE" || { - test "$CUR_FIX" -lt "$MIN_FIX" && die_upgrade "$VERS_LEAST" - true - } - } - } -} diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 7daa66ee7..fa89bf632 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -334,7 +334,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment } // Use hybrid migration strategy that intelligently combines external and embedded migrations - err = migrations.RunHybridMigrations(cctx.Context(), version.RepoVersion, cctx.ConfigRoot, false) + // Use req.Context instead of cctx.Context() to avoid attempting repo open before migrations complete + err = migrations.RunHybridMigrations(req.Context, version.RepoVersion, cctx.ConfigRoot, false) if err != nil { fmt.Println("Repository migration failed:") fmt.Printf(" %s\n", err) @@ -387,7 +388,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment log.Errorf("failed to create autoconf client: %v", err) } else { // Start primes cache and starts background updater - if _, err := client.Start(cctx.Context()); err != nil { + // Use req.Context for background updater lifecycle (node doesn't exist yet) + if _, err := client.Start(req.Context); err != nil { log.Errorf("failed to start autoconf updater: %v", err) } } diff --git a/core/commands/repo.go b/core/commands/repo.go index 017143127..622e92d7e 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -423,19 +423,12 @@ migration. Versions below 16 require external migration tools. return fmt.Errorf("downgrade from version %d to %d requires --allow-downgrade flag", currentVersion, targetVersion) } - // Check if repo is locked by daemon before running migration - locked, err := fsrepo.LockedByOtherProcess(cctx.ConfigRoot) - if err != nil { - return fmt.Errorf("could not check repo lock: %w", err) - } - if locked { - return fmt.Errorf("cannot run migration while daemon is running (repo.lock exists)") - } - fmt.Printf("Migrating repository from version %d to %d...\n", currentVersion, targetVersion) // Use hybrid migration strategy that intelligently combines external and embedded migrations - err = migrations.RunHybridMigrations(cctx.Context(), targetVersion, cctx.ConfigRoot, allowDowngrade) + // Use req.Context instead of cctx.Context() to avoid opening the repo before migrations run, + // which would acquire the lock that migrations need + err = migrations.RunHybridMigrations(req.Context, targetVersion, cctx.ConfigRoot, allowDowngrade) if err != nil { fmt.Println("Repository migration failed:") fmt.Printf(" %s\n", err) diff --git a/docs/changelogs/v0.38.md b/docs/changelogs/v0.38.md index 2edb31adf..2bffa5885 100644 --- a/docs/changelogs/v0.38.md +++ b/docs/changelogs/v0.38.md @@ -5,6 +5,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.38.0](#v0380) +- [v0.38.1](#v0381) ## v0.38.0 @@ -290,3 +291,34 @@ The new [`Internal.MFSNoFlushLimit`](https://github.com/ipfs/kubo/blob/master/do | Jakub Sztandera | 1 | +67/-15 | 3 | | Masih H. Derkani | 1 | +1/-2 | 2 | | Dominic Della Valle | 1 | +2/-1 | 1 | + +## v0.38.1 + +Fixes migration panic on Windows when upgrading from v0.37 to v0.38 ("panic: error can't be dealt with transactionally: Access is denied"). + +Updates go-ds-pebble to v0.5.3 (pebble v2.1.0). + +### ๐Ÿ“ Changelog + +
Full Changelog + +- github.com/ipfs/kubo: + - chore: v0.38.1 + - fix: migrations for Windows (#11010) ([ipfs/kubo#11010](https://github.com/ipfs/kubo/pull/11010)) + - Upgrade go-ds-pebble to v0.5.3 (#11011) ([ipfs/kubo#11011](https://github.com/ipfs/kubo/pull/11011)) + - upgrade go-ds-pebble to v0.5.2 (#11000) ([ipfs/kubo#11000](https://github.com/ipfs/kubo/pull/11000)) +- github.com/ipfs/go-ds-pebble (v0.5.1 -> v0.5.3): + - new version (#62) ([ipfs/go-ds-pebble#62](https://github.com/ipfs/go-ds-pebble/pull/62)) + - fix panic when batch is reused after commit (#61) ([ipfs/go-ds-pebble#61](https://github.com/ipfs/go-ds-pebble/pull/61)) + - new version (#60) ([ipfs/go-ds-pebble#60](https://github.com/ipfs/go-ds-pebble/pull/60)) + - Upgrade to pebble v2.1.0 (#59) ([ipfs/go-ds-pebble#59](https://github.com/ipfs/go-ds-pebble/pull/59)) + - update readme (#57) ([ipfs/go-ds-pebble#57](https://github.com/ipfs/go-ds-pebble/pull/57)) + +
+ +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors + +| Contributor | Commits | Lines ยฑ | Files Changed | +|-------------|---------|---------|---------------| +| Marcin Rataj | 2 | +613/-267 | 15 | +| Andrew Gillis | 6 | +148/-22 | 8 | diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 81c2a147b..e383e1f25 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -16,8 +16,10 @@ require ( require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -29,11 +31,10 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble/v2 v2.0.6 // indirect + github.com/cockroachdb/pebble/v2 v2.1.0 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -81,7 +82,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect - github.com/ipfs/go-ds-pebble v0.5.1 // indirect + github.com/ipfs/go-ds-pebble v0.5.3 // indirect github.com/ipfs/go-dsqueue v0.0.5 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect @@ -132,6 +133,7 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index eb90aa12a..553044987 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -29,11 +29,15 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -81,20 +85,18 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -319,8 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ= -github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0= +github.com/ipfs/go-ds-pebble v0.5.3 h1:4esRt82+LkenUnIWyUCghR1gzRfqeCYGGKX/hRmabro= +github.com/ipfs/go-ds-pebble v0.5.3/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= @@ -490,6 +492,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= diff --git a/go.mod b/go.mod index a1a2a3ab1..48702ed0f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/ceramicnetwork/go-dag-jose v0.1.1 github.com/cheggaaa/pb v1.0.29 - github.com/cockroachdb/pebble/v2 v2.0.6 + github.com/cockroachdb/pebble/v2 v2.1.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dustin/go-humanize v1.0.1 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 @@ -32,7 +32,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 - github.com/ipfs/go-ds-pebble v0.5.1 + github.com/ipfs/go-ds-pebble v0.5.3 github.com/ipfs/go-fs-lock v0.1.1 github.com/ipfs/go-ipfs-cmds v0.15.0 github.com/ipfs/go-ipld-cbor v0.2.1 @@ -97,8 +97,10 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -107,9 +109,8 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect @@ -180,6 +181,7 @@ require ( github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect diff --git a/go.sum b/go.sum index 5dd61e2dc..0cd74a1d1 100644 --- a/go.sum +++ b/go.sum @@ -47,12 +47,16 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -112,20 +116,18 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -386,8 +388,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ= -github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0= +github.com/ipfs/go-ds-pebble v0.5.3 h1:4esRt82+LkenUnIWyUCghR1gzRfqeCYGGKX/hRmabro= +github.com/ipfs/go-ds-pebble v0.5.3/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= @@ -586,6 +588,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= diff --git a/mk/golang.mk b/mk/golang.mk index 4f4cd4fed..b50179a0a 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -1,5 +1,4 @@ # golang utilities -GO_MIN_VERSION = 1.25 export GO111MODULE=on @@ -74,11 +73,8 @@ test_go_lint: test/bin/golangci-lint test_go: $(TEST_GO) -check_go_version: - @$(GOCC) version - bin/check_go_version $(GO_MIN_VERSION) +# Version check is no longer needed - go.mod enforces minimum version .PHONY: check_go_version -DEPS_GO += check_go_version TEST += $(TEST_GO) TEST_SHORT += test_go_fmt test_go_short diff --git a/plugin/plugins/pebbleds/pebbleds.go b/plugin/plugins/pebbleds/pebbleds.go index fab1cc16a..141eff74b 100644 --- a/plugin/plugins/pebbleds/pebbleds.go +++ b/plugin/plugins/pebbleds/pebbleds.go @@ -134,7 +134,7 @@ func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { WALBytesPerSync: walBytesPerSync, } if maxConcurrentCompactions != 0 { - c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions } + c.pebbleOpts.CompactionConcurrencyRange = func() (int, int) { return 1, maxConcurrentCompactions } } if walMinSyncSec != 0 { c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second } diff --git a/repo/fsrepo/migrations/atomicfile/atomicfile.go b/repo/fsrepo/migrations/atomicfile/atomicfile.go index 87704196d..209b8c368 100644 --- a/repo/fsrepo/migrations/atomicfile/atomicfile.go +++ b/repo/fsrepo/migrations/atomicfile/atomicfile.go @@ -1,6 +1,7 @@ package atomicfile import ( + "fmt" "io" "os" "path/filepath" @@ -34,23 +35,27 @@ func New(path string, mode os.FileMode) (*File, error) { // Close atomically replaces the target file with the temporary file func (f *File) Close() error { - if err := f.File.Close(); err != nil { - os.Remove(f.File.Name()) - return err + closeErr := f.File.Close() + if closeErr != nil { + // Try to cleanup temp file, but prioritize close error + _ = os.Remove(f.File.Name()) + return closeErr } - - if err := os.Rename(f.File.Name(), f.path); err != nil { - os.Remove(f.File.Name()) - return err - } - - return nil + return os.Rename(f.File.Name(), f.path) } // Abort removes the temporary file without replacing the target func (f *File) Abort() error { - f.File.Close() - return os.Remove(f.File.Name()) + closeErr := f.File.Close() + removeErr := os.Remove(f.File.Name()) + + if closeErr != nil && removeErr != nil { + return fmt.Errorf("abort failed: close: %w, remove: %v", closeErr, removeErr) + } + if closeErr != nil { + return closeErr + } + return removeErr } // ReadFrom reads from the given reader into the atomic file diff --git a/repo/fsrepo/migrations/atomicfile/atomicfile_test.go b/repo/fsrepo/migrations/atomicfile/atomicfile_test.go new file mode 100644 index 000000000..668045d12 --- /dev/null +++ b/repo/fsrepo/migrations/atomicfile/atomicfile_test.go @@ -0,0 +1,208 @@ +package atomicfile + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNew_Success verifies atomic file creation +func TestNew_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + defer func() { _ = af.Abort() }() + + // Verify temp file exists + assert.FileExists(t, af.File.Name()) + + // Verify temp file is in same directory + assert.Equal(t, dir, filepath.Dir(af.File.Name())) +} + +// TestClose_Success verifies atomic replacement +func TestClose_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + content := []byte("test content") + _, err = af.Write(content) + require.NoError(t, err) + + tempName := af.File.Name() + + require.NoError(t, af.Close()) + + // Verify target file exists with correct content + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, content, data) + + // Verify temp file removed + assert.NoFileExists(t, tempName) +} + +// TestAbort_Success verifies cleanup +func TestAbort_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + require.NoError(t, af.Abort()) + + // Verify temp file removed + assert.NoFileExists(t, tempName) + + // Verify target not created + assert.NoFileExists(t, path) +} + +// TestAbort_ErrorHandling tests error capture +func TestAbort_ErrorHandling(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + // Close file to force close error + af.File.Close() + + // Remove temp file to force remove error + os.Remove(af.File.Name()) + + err = af.Abort() + // Should get both errors + require.Error(t, err) + assert.Contains(t, err.Error(), "abort failed") +} + +// TestClose_CloseError verifies cleanup on close failure +func TestClose_CloseError(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + // Close file to force close error + af.File.Close() + + err = af.Close() + require.Error(t, err) + + // Verify temp file cleaned up even on error + assert.NoFileExists(t, tempName) +} + +// TestReadFrom verifies io.Copy integration +func TestReadFrom(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + defer func() { _ = af.Abort() }() + + content := []byte("test content from reader") + n, err := af.ReadFrom(bytes.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, int64(len(content)), n) +} + +// TestFilePermissions verifies mode is set correctly +func TestFilePermissions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0600) + require.NoError(t, err) + + _, err = af.Write([]byte("test")) + require.NoError(t, err) + + require.NoError(t, af.Close()) + + info, err := os.Stat(path) + require.NoError(t, err) + + // On Unix, check exact permissions + if runtime.GOOS != "windows" { + mode := info.Mode().Perm() + assert.Equal(t, os.FileMode(0600), mode) + } +} + +// TestMultipleAbortsSafe verifies calling Abort multiple times is safe +func TestMultipleAbortsSafe(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + // First abort should succeed + require.NoError(t, af.Abort()) + assert.NoFileExists(t, tempName, "temp file should be removed after first abort") + + // Second abort should handle gracefully (file already gone) + err = af.Abort() + // Error is acceptable since file is already removed, but it should not panic + t.Logf("Second Abort() returned: %v", err) +} + +// TestNoTempFilesAfterOperations verifies no .tmp-* files remain after operations +func TestNoTempFilesAfterOperations(t *testing.T) { + const testIterations = 5 + + tests := []struct { + name string + operation func(*File) error + }{ + {"close", (*File).Close}, + {"abort", (*File).Abort}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + + // Perform multiple operations + for i := 0; i < testIterations; i++ { + path := filepath.Join(dir, fmt.Sprintf("test%d.txt", i)) + + af, err := New(path, 0644) + require.NoError(t, err) + + _, err = af.Write([]byte("test data")) + require.NoError(t, err) + + require.NoError(t, tt.operation(af)) + } + + // Check for any .tmp-* files + tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) + require.NoError(t, err) + assert.Empty(t, tmpFiles, "should be no temp files after %s", tt.name) + }) + } +} diff --git a/repo/fsrepo/migrations/common/utils.go b/repo/fsrepo/migrations/common/utils.go index 217da609f..e7d704dad 100644 --- a/repo/fsrepo/migrations/common/utils.go +++ b/repo/fsrepo/migrations/common/utils.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "encoding/json" "fmt" "io" @@ -40,47 +41,51 @@ func Must(err error) { // WithBackup performs a config file operation with automatic backup and rollback on error func WithBackup(configPath string, backupSuffix string, fn func(in io.ReadSeeker, out io.Writer) error) error { - in, err := os.Open(configPath) + // Read the entire file into memory first + // This allows us to close the file before doing atomic operations, + // which is necessary on Windows where open files can't be renamed + data, err := os.ReadFile(configPath) if err != nil { - return err + return fmt.Errorf("failed to read config file %s: %w", configPath, err) } - defer in.Close() - // Create backup - backup, err := atomicfile.New(configPath+backupSuffix, 0600) + // Create an in-memory reader for the data + in := bytes.NewReader(data) + + // Create backup atomically to prevent partial backup on interruption + backupPath := configPath + backupSuffix + backup, err := atomicfile.New(backupPath, 0600) if err != nil { - return err + return fmt.Errorf("failed to create backup file for %s: %w", backupPath, err) } - - // Copy to backup - if _, err := backup.ReadFrom(in); err != nil { + if _, err := backup.Write(data); err != nil { Must(backup.Abort()) - return err + return fmt.Errorf("failed to write backup data: %w", err) } - - // Reset input for reading - if _, err := in.Seek(0, io.SeekStart); err != nil { + if err := backup.Close(); err != nil { Must(backup.Abort()) - return err + return fmt.Errorf("failed to finalize backup: %w", err) } - // Create output file + // Create output file atomically out, err := atomicfile.New(configPath, 0600) if err != nil { - Must(backup.Abort()) - return err + // Clean up backup on error + os.Remove(backupPath) + return fmt.Errorf("failed to create atomic file for %s: %w", configPath, err) } // Run the conversion function if err := fn(in, out); err != nil { Must(out.Abort()) - Must(backup.Abort()) - return err + // Clean up backup on error + os.Remove(backupPath) + return fmt.Errorf("config conversion failed: %w", err) } - // Close everything on success + // Close the output file atomically Must(out.Close()) - Must(backup.Close()) + // Backup remains for potential revert return nil } diff --git a/repo/fsrepo/migrations/embedded.go b/repo/fsrepo/migrations/embedded.go index a2aa4d252..a8218be63 100644 --- a/repo/fsrepo/migrations/embedded.go +++ b/repo/fsrepo/migrations/embedded.go @@ -6,6 +6,7 @@ import ( "log" "os" + lockfile "github.com/ipfs/go-fs-lock" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration" mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration" @@ -109,6 +110,13 @@ func RunEmbeddedMigrations(ctx context.Context, targetVer int, ipfsDir string, a return err } + // Acquire lock once for all embedded migrations to prevent concurrent access + lk, err := lockfile.Lock(ipfsDir, "repo.lock") + if err != nil { + return fmt.Errorf("failed to acquire repo lock: %w", err) + } + defer lk.Close() + fromVer, err := RepoVersion(ipfsDir) if err != nil { return fmt.Errorf("could not get repo version: %w", err) diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index c94ebc586..c18721bae 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -11,6 +11,8 @@ import ( func TestRepoDir(t *testing.T) { fakeHome := t.TempDir() t.Setenv("HOME", fakeHome) + // On Windows, os.UserHomeDir() uses USERPROFILE, not HOME + t.Setenv("USERPROFILE", fakeHome) fakeIpfs := filepath.Join(fakeHome, ".ipfs") t.Setenv(config.EnvDir, fakeIpfs) diff --git a/test/cli/migrations/migration_16_to_latest_test.go b/test/cli/migrations/migration_16_to_latest_test.go index 521b31646..97a1ec1ff 100644 --- a/test/cli/migrations/migration_16_to_latest_test.go +++ b/test/cli/migrations/migration_16_to_latest_test.go @@ -21,6 +21,7 @@ import ( ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -52,6 +53,13 @@ func TestMigration16ToLatest(t *testing.T) { // Comparison tests using 'ipfs repo migrate' command t.Run("repo migrate: forward migration with auto values", testRepoMigrationWithAuto) t.Run("repo migrate: backward migration", testRepoBackwardMigration) + + // Temp file and backup cleanup tests + t.Run("daemon migrate: no temp files after successful migration", testNoTempFilesAfterSuccessfulMigration) + t.Run("daemon migrate: no temp files after failed migration", testNoTempFilesAfterFailedMigration) + t.Run("daemon migrate: backup files persist after successful migration", testBackupFilesPersistAfterSuccessfulMigration) + t.Run("repo migrate: backup files can revert migration", testBackupFilesCanRevertMigration) + t.Run("repo migrate: conversion failure cleans up temp files", testConversionFailureCleanup) } // ============================================================================= @@ -392,10 +400,15 @@ func setupStaticV16Repo(t *testing.T) *harness.Node { v16FixturePath := "testdata/v16-repo" // Create a temporary test directory - each test gets its own copy - // Use ./tmp.DELETEME/ as requested by user instead of /tmp/ - tmpDir := filepath.Join("tmp.DELETEME", "migration-test-"+t.Name()) + // Sanitize test name for Windows - replace invalid characters + sanitizedName := strings.Map(func(r rune) rune { + if strings.ContainsRune(`<>:"/\|?*`, r) { + return '_' + } + return r + }, t.Name()) + tmpDir := filepath.Join(t.TempDir(), "migration-test-"+sanitizedName) require.NoError(t, os.MkdirAll(tmpDir, 0755)) - t.Cleanup(func() { os.RemoveAll(tmpDir) }) // Convert to absolute path for harness absTmpDir, err := filepath.Abs(tmpDir) @@ -559,6 +572,8 @@ func testRepoBackwardMigration(t *testing.T) { // First run forward migration to get to v17 result := node.RunIPFS("repo", "migrate") + t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) + t.Logf("Forward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Forward migration should succeed") // Verify we're at the latest version @@ -569,6 +584,8 @@ func testRepoBackwardMigration(t *testing.T) { // Now run reverse migration back to v16 result = node.RunIPFS("repo", "migrate", "--to=16", "--allow-downgrade") + t.Logf("Backward migration stdout:\n%s", result.Stdout.String()) + t.Logf("Backward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Reverse migration should succeed") // Verify version was downgraded to 16 @@ -753,3 +770,149 @@ func runDaemonWithMultipleMigrationMonitoring(t *testing.T, node *harness.Node, } } } + +// ============================================================================= +// TEMP FILE AND BACKUP CLEANUP TESTS +// ============================================================================= + +// Helper functions for test cleanup assertions +func assertNoTempFiles(t *testing.T, dir string, msgAndArgs ...interface{}) { + t.Helper() + tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) + require.NoError(t, err) + assert.Empty(t, tmpFiles, msgAndArgs...) +} + +func backupPath(configPath string, fromVer, toVer int) string { + return fmt.Sprintf("%s.%d-to-%d.bak", configPath, fromVer, toVer) +} + +func setupDaemonCmd(ctx context.Context, node *harness.Node, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, node.IPFSBin, args...) + cmd.Dir = node.Dir + for k, v := range node.Runner.Env { + cmd.Env = append(cmd.Env, k+"="+v) + } + return cmd +} + +func testNoTempFilesAfterSuccessfulMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Run successful migration + _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) + require.True(t, migrationSuccess, "migration should succeed") + + assertNoTempFiles(t, node.Dir, "no temp files should remain after successful migration") +} + +func testNoTempFilesAfterFailedMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Corrupt config to force migration failure + configPath := filepath.Join(node.Dir, "config") + corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON + require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) + + // Attempt migration (should fail) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cmd := setupDaemonCmd(ctx, node, "daemon", "--migrate") + output, _ := cmd.CombinedOutput() + t.Logf("Failed migration output: %s", output) + + assertNoTempFiles(t, node.Dir, "no temp files should remain after failed migration") +} + +func testBackupFilesPersistAfterSuccessfulMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Run migration from v16 to latest (v18) + _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) + require.True(t, migrationSuccess, "migration should succeed") + + // Check for backup files from each migration step + configPath := filepath.Join(node.Dir, "config") + backup16to17 := backupPath(configPath, 16, 17) + backup17to18 := backupPath(configPath, 17, 18) + + // Both backup files should exist + assert.FileExists(t, backup16to17, "16-to-17 backup should exist") + assert.FileExists(t, backup17to18, "17-to-18 backup should exist") + + // Verify backup files contain valid JSON + data16to17, err := os.ReadFile(backup16to17) + require.NoError(t, err) + var config16to17 map[string]interface{} + require.NoError(t, json.Unmarshal(data16to17, &config16to17), "16-to-17 backup should be valid JSON") + + data17to18, err := os.ReadFile(backup17to18) + require.NoError(t, err) + var config17to18 map[string]interface{} + require.NoError(t, json.Unmarshal(data17to18, &config17to18), "17-to-18 backup should be valid JSON") +} + +func testBackupFilesCanRevertMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + configPath := filepath.Join(node.Dir, "config") + versionPath := filepath.Join(node.Dir, "version") + + // Read original v16 config + originalConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Migrate to v17 only + result := node.RunIPFS("repo", "migrate", "--to=17") + require.Empty(t, result.Stderr.String(), "migration to v17 should succeed") + + // Verify backup exists + backup16to17 := backupPath(configPath, 16, 17) + assert.FileExists(t, backup16to17, "16-to-17 backup should exist") + + // Manually revert using backup + backupData, err := os.ReadFile(backup16to17) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, backupData, 0600)) + require.NoError(t, os.WriteFile(versionPath, []byte("16"), 0644)) + + // Verify config matches original + revertedConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + assert.JSONEq(t, string(originalConfig), string(revertedConfig), "reverted config should match original") + + // Verify version is back to 16 + versionData, err := os.ReadFile(versionPath) + require.NoError(t, err) + assert.Equal(t, "16", strings.TrimSpace(string(versionData)), "version should be reverted to 16") +} + +func testConversionFailureCleanup(t *testing.T) { + // This test verifies that when a migration's conversion function fails, + // all temporary files are cleaned up properly + node := setupStaticV16Repo(t) + + configPath := filepath.Join(node.Dir, "config") + + // Create a corrupted config that will cause conversion to fail during JSON parsing + // The migration will read this, attempt to parse as JSON, and fail + corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON - missing closing bracket + require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) + + // Attempt migration (should fail during conversion) + result := node.RunIPFS("repo", "migrate") + require.NotEmpty(t, result.Stderr.String(), "migration should fail with error") + + assertNoTempFiles(t, node.Dir, "no temp files should remain after conversion failure") + + // Verify no backup files were created (failure happened before backup) + backupFiles, err := filepath.Glob(filepath.Join(node.Dir, "config.*.bak")) + require.NoError(t, err) + assert.Empty(t, backupFiles, "no backup files should be created on conversion failure") + + // Verify corrupted config is unchanged (atomic operations prevented overwrite) + currentConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + assert.Equal(t, corruptedJson, string(currentConfig), "corrupted config should remain unchanged") +} diff --git a/test/cli/migrations/migration_concurrent_test.go b/test/cli/migrations/migration_concurrent_test.go new file mode 100644 index 000000000..8c716f51c --- /dev/null +++ b/test/cli/migrations/migration_concurrent_test.go @@ -0,0 +1,55 @@ +package migrations + +// NOTE: These concurrent migration tests require the local Kubo binary (built with 'make build') to be in PATH. +// +// To run these tests successfully: +// export PATH="$(pwd)/cmd/ipfs:$PATH" +// go test ./test/cli/migrations/ + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const daemonStartupWait = 2 * time.Second + +// TestConcurrentMigrations tests concurrent daemon --migrate attempts +func TestConcurrentMigrations(t *testing.T) { + t.Parallel() + + t.Run("concurrent daemon migrations prevented by lock", testConcurrentDaemonMigrations) +} + +func testConcurrentDaemonMigrations(t *testing.T) { + node := setupStaticV16Repo(t) + + // Start first daemon --migrate in background (holds repo.lock) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + firstDaemon := setupDaemonCmd(ctx, node, "daemon", "--migrate") + require.NoError(t, firstDaemon.Start()) + defer func() { + // Shutdown first daemon + shutdownCmd := setupDaemonCmd(context.Background(), node, "shutdown") + _ = shutdownCmd.Run() + _ = firstDaemon.Wait() + }() + + // Wait for first daemon to start and acquire lock + time.Sleep(daemonStartupWait) + + // Attempt second daemon --migrate (should fail due to lock) + secondDaemon := setupDaemonCmd(context.Background(), node, "daemon", "--migrate") + output, err := secondDaemon.CombinedOutput() + t.Logf("Second daemon output: %s", output) + + // Should fail with lock error + require.Error(t, err, "second daemon should fail when first daemon holds lock") + require.Contains(t, string(output), "lock", "error should mention lock") + + assertNoTempFiles(t, node.Dir, "no temp files should be created when lock fails") +} diff --git a/test/cli/migrations/migration_mixed_15_to_latest_test.go b/test/cli/migrations/migration_mixed_15_to_latest_test.go index 9f1a482f8..6ee96b939 100644 --- a/test/cli/migrations/migration_mixed_15_to_latest_test.go +++ b/test/cli/migrations/migration_mixed_15_to_latest_test.go @@ -23,8 +23,9 @@ import ( "os" "os/exec" "path/filepath" + "runtime" + "slices" "strings" - "syscall" "testing" "time" @@ -61,7 +62,8 @@ func testDaemonMigration15ToLatest(t *testing.T) { node := setupStaticV15Repo(t) // Create mock migration binary for 15โ†’16 (16โ†’17 will use embedded migration) - createMockMigrationBinary(t, "15", "16") + mockBinDir := createMockMigrationBinary(t, "15", "16") + customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") @@ -80,7 +82,7 @@ func testDaemonMigration15ToLatest(t *testing.T) { originalPeerID := getNestedValue(originalConfig, "Identity.PeerID") // Run dual migration using daemon --migrate - stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node) + stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node, customPath) // Debug output t.Logf("Daemon output:\n%s", stdoutOutput) @@ -124,7 +126,8 @@ func testRepoMigration15ToLatest(t *testing.T) { node := setupStaticV15Repo(t) // Create mock migration binary for 15โ†’16 (16โ†’17 will use embedded migration) - createMockMigrationBinary(t, "15", "16") + mockBinDir := createMockMigrationBinary(t, "15", "16") + customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") @@ -135,16 +138,7 @@ func testRepoMigration15ToLatest(t *testing.T) { require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15") // Run migration using 'ipfs repo migrate' with custom PATH - result := node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify final version is latest @@ -184,10 +178,10 @@ func setupStaticV15Repo(t *testing.T) *harness.Node { } // runDaemonWithLegacyMigrationMonitoring monitors for hybrid migration patterns -func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node) (string, bool) { +func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node, customPath string) (string, bool) { // Monitor for hybrid migration completion - use "Hybrid migration completed successfully" as success pattern stdoutOutput, daemonStarted := runDaemonWithMigrationMonitoringCustomEnv(t, node, "Using hybrid migration strategy", "Hybrid migration completed successfully", map[string]string{ - "PATH": os.Getenv("PATH"), // Pass current PATH which includes our mock binaries + "PATH": customPath, // Pass custom PATH with our mock binaries }) // Check for hybrid migration patterns in output @@ -271,17 +265,59 @@ func runDaemonWithMigrationMonitoringCustomEnv(t *testing.T, node *harness.Node, t.Log("Daemon startup timed out") } - // Stop the daemon + // Stop the daemon using ipfs shutdown command for graceful shutdown if cmd.Process != nil { - _ = cmd.Process.Signal(syscall.SIGTERM) + shutdownCmd := exec.Command(node.IPFSBin, "shutdown") + shutdownCmd.Dir = node.Dir + for k, v := range node.Runner.Env { + shutdownCmd.Env = append(shutdownCmd.Env, k+"="+v) + } + + if err := shutdownCmd.Run(); err != nil { + // If graceful shutdown fails, force kill + _ = cmd.Process.Kill() + } + + // Wait for process to exit _ = cmd.Wait() } return outputBuffer.String(), daemonReady && migrationStarted && migrationCompleted } -// createMockMigrationBinary creates a platform-agnostic Go binary for migration on PATH -func createMockMigrationBinary(t *testing.T, fromVer, toVer string) { +// buildCustomPath creates a custom PATH with mock migration binaries prepended. +// This is necessary for test isolation when running tests in parallel with t.Parallel(). +// Without isolated PATH handling, parallel tests can interfere with each other through +// global PATH modifications, causing tests to download real migration binaries instead +// of using the test mocks. +func buildCustomPath(mockBinDirs ...string) string { + // Prepend mock directories to ensure they're found first + pathElements := append(mockBinDirs, os.Getenv("PATH")) + return strings.Join(pathElements, string(filepath.ListSeparator)) +} + +// runMigrationWithCustomPath runs a migration command with a custom PATH environment. +// This ensures the migration uses our mock binaries instead of downloading real ones. +func runMigrationWithCustomPath(node *harness.Node, customPath string, args ...string) *harness.RunResult { + return node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: args, + CmdOpts: []harness.CmdOpt{ + func(cmd *exec.Cmd) { + // Remove existing PATH entries using slices.DeleteFunc + cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool { + return strings.HasPrefix(s, "PATH=") + }) + // Add custom PATH + cmd.Env = append(cmd.Env, "PATH="+customPath) + }, + }, + }) +} + +// createMockMigrationBinary creates a platform-agnostic Go binary for migration testing. +// Returns the directory containing the binary to be added to PATH. +func createMockMigrationBinary(t *testing.T, fromVer, toVer string) string { // Create bin directory for migration binaries binDir := t.TempDir() @@ -289,73 +325,60 @@ func createMockMigrationBinary(t *testing.T, fromVer, toVer string) { scriptName := fmt.Sprintf("fs-repo-%s-to-%s", fromVer, toVer) sourceFile := filepath.Join(binDir, scriptName+".go") binaryPath := filepath.Join(binDir, scriptName) + if runtime.GOOS == "windows" { + binaryPath += ".exe" + } + // Generate minimal mock migration binary code goSource := fmt.Sprintf(`package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - +import ("fmt"; "os"; "path/filepath"; "strings"; "time") func main() { - // Parse command line arguments - real migration binaries expect -path= - var repoPath string + var path string var revert bool - for _, arg := range os.Args[1:] { - if strings.HasPrefix(arg, "-path=") { - repoPath = strings.TrimPrefix(arg, "-path=") - } else if arg == "-revert" { - revert = true - } + for _, a := range os.Args[1:] { + if strings.HasPrefix(a, "-path=") { path = a[6:] } + if a == "-revert" { revert = true } } - - if repoPath == "" { - fmt.Fprintf(os.Stderr, "Usage: %%s -path= [-verbose=true] [-revert]\n", os.Args[0]) + if path == "" { fmt.Fprintln(os.Stderr, "missing -path="); os.Exit(1) } + + from, to := "%s", "%s" + if revert { from, to = to, from } + fmt.Printf("fake applying %%s-to-%%s repo migration\n", from, to) + + // Create and immediately remove lock file to simulate proper locking behavior + lockPath := filepath.Join(path, "repo.lock") + lockFile, err := os.Create(lockPath) + if err != nil && !os.IsExist(err) { + fmt.Fprintf(os.Stderr, "Error creating lock: %%v\n", err) os.Exit(1) } - - // Determine source and target versions based on revert flag - var sourceVer, targetVer string - if revert { - // When reverting, we go backwards: fs-repo-15-to-16 with -revert goes 16โ†’15 - sourceVer = "%s" - targetVer = "%s" - } else { - // Normal forward migration: fs-repo-15-to-16 goes 15โ†’16 - sourceVer = "%s" - targetVer = "%s" + if lockFile != nil { + lockFile.Close() + defer os.Remove(lockPath) } - - // Print migration message (same format as real migrations) - fmt.Printf("fake applying %%s-to-%%s repo migration\n", sourceVer, targetVer) - - // Update version file - versionFile := filepath.Join(repoPath, "version") - err := os.WriteFile(versionFile, []byte(targetVer), 0644) - if err != nil { - fmt.Fprintf(os.Stderr, "Error updating version: %%v\n", err) + + // Small delay to simulate migration work + time.Sleep(10 * time.Millisecond) + + if err := os.WriteFile(filepath.Join(path, "version"), []byte(to), 0644); err != nil { + fmt.Fprintf(os.Stderr, "Error: %%v\n", err) os.Exit(1) } -} -`, toVer, fromVer, fromVer, toVer) +}`, fromVer, toVer) require.NoError(t, os.WriteFile(sourceFile, []byte(goSource), 0644)) // Compile the Go binary - require.NoError(t, os.Setenv("CGO_ENABLED", "0")) // Ensure static binary - require.NoError(t, exec.Command("go", "build", "-o", binaryPath, sourceFile).Run()) - - // Add bin directory to PATH for this test - currentPath := os.Getenv("PATH") - newPath := binDir + string(filepath.ListSeparator) + currentPath - require.NoError(t, os.Setenv("PATH", newPath)) - t.Cleanup(func() { os.Setenv("PATH", currentPath) }) + cmd := exec.Command("go", "build", "-o", binaryPath, sourceFile) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") // Ensure static binary + require.NoError(t, cmd.Run()) // Verify the binary exists and is executable _, err := os.Stat(binaryPath) require.NoError(t, err, "Mock binary should exist") + + // Return the bin directory to be added to PATH + return binDir } // expectedMigrationSteps generates the expected migration step strings for a version range. @@ -416,26 +439,19 @@ func testRepoReverseHybridMigrationLatestTo15(t *testing.T) { // Start with v15 fixture and migrate forward to latest to create proper backup files node := setupStaticV15Repo(t) - // Create mock migration binary for 15โ†’16 (needed for forward migration) - createMockMigrationBinary(t, "15", "16") - // Create mock migration binary for 16โ†’15 (needed for downgrade) - createMockMigrationBinary(t, "16", "15") + // Create mock migration binaries for both forward and reverse migrations + mockBinDirs := []string{ + createMockMigrationBinary(t, "15", "16"), // for forward migration + createMockMigrationBinary(t, "16", "15"), // for downgrade + } + customPath := buildCustomPath(mockBinDirs...) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Step 1: Forward migration from v15 to latest to create backup files t.Logf("Step 1: Forward migration v15 โ†’ v%d", ipfs.RepoVersion) - result := node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") // Debug: print the output to see what happened t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) @@ -459,16 +475,7 @@ func testRepoReverseHybridMigrationLatestTo15(t *testing.T) { // Step 2: Reverse hybrid migration from latest to v15 t.Logf("Step 2: Reverse hybrid migration v%d โ†’ v15", ipfs.RepoVersion) - result = node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate", "--to=15", "--allow-downgrade"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result = runMigrationWithCustomPath(node, customPath, "repo", "migrate", "--to=15", "--allow-downgrade") require.Empty(t, result.Stderr.String(), "Reverse hybrid migration should succeed without errors") // Debug output diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 65d3151aa..268ccae27 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -28,12 +28,14 @@ require ( github.com/Antonboom/testifylint v1.5.2 // indirect github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/Crocmagnon/fatcontext v0.7.1 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexkohler/nakedret/v2 v2.0.5 // indirect @@ -60,11 +62,10 @@ require ( github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/ckaznocha/intrange v0.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble/v2 v2.0.6 // indirect + github.com/cockroachdb/pebble/v2 v2.1.0 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -202,6 +203,7 @@ require ( github.com/mgechev/revive v1.7.0 // indirect github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/miekg/dns v1.1.68 // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index aec72c23d..4c3e98cce 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -27,8 +27,8 @@ github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/k github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Crocmagnon/fatcontext v0.7.1 h1:SC/VIbRRZQeQWj/TcQBS6JmrXcfA+BU4OGSVUt54PjM= github.com/Crocmagnon/fatcontext v0.7.1/go.mod h1:1wMvv3NXEBJucFGfwOJBxSVWcoIO6emV215SMkW9MFU= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 h1:Sz1JIXEcSfhz7fUi7xHnhpIE0thVASYjvosApmHuD2k= @@ -41,6 +41,10 @@ github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+ github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= @@ -104,20 +108,18 @@ github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+U github.com/ckaznocha/intrange v0.3.0 h1:VqnxtK32pxgkhJgYQEeOArVidIPg+ahLP7WBOXZd5ZY= github.com/ckaznocha/intrange v0.3.0/go.mod h1:+I/o2d2A1FBHgGELbGxzIcyd3/9l9DuwjM8FsbSS3Lo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -522,6 +524,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= diff --git a/version.go b/version.go index eb1dd3850..9ebe9f187 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal. -const CurrentVersionNumber = "0.38.0" +const CurrentVersionNumber = "0.38.1" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint