Add 'ipfs repo migrate' command (#8428)

* Add 'ipfs repo migrate' command

This PR replaces #7658 that was originally contributed by zaibons, in order to move code into a branch and avoid some CI problem.

The command allows the user to run the repo migration without starting the daemon.

resolves #7471

* return non-ErrNeedMigration errors from fsrepo.Open()

Co-authored-by: Gus Eggert <gus@gus.dev>

core/commands/commands_test.go

@@ -217,6 +217,7 @@ func TestCommands(t *testing.T) {
 		"/repo",
 		"/repo/fsck",
 		"/repo/gc",
+		"/repo/migrate",
 		"/repo/stat",
 		"/repo/verify",
 		"/repo/version",

core/commands/repo.go

@@ -11,11 +11,14 @@ import (
 	"sync"
 	"text/tabwriter"
 
-	humanize "github.com/dustin/go-humanize"
+	oldcmds "github.com/ipfs/go-ipfs/commands"
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
+	"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
+	"github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher"
 
+	humanize "github.com/dustin/go-humanize"
 	cid "github.com/ipfs/go-cid"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
 	cmds "github.com/ipfs/go-ipfs-cmds"
@@ -39,6 +42,7 @@ var RepoCmd = &cmds.Command{
 		"fsck":    repoFsckCmd,
 		"version": repoVersionCmd,
 		"verify":  repoVerifyCmd,
+		"migrate": repoMigrateCmd,
 	},
 }
 
@@ -49,9 +53,10 @@ type GcResult struct {
 }
 
 const (
-	repoStreamErrorsOptionName = "stream-errors"
-	repoQuietOptionName        = "quiet"
-	repoSilentOptionName       = "silent"
+	repoStreamErrorsOptionName   = "stream-errors"
+	repoQuietOptionName          = "quiet"
+	repoSilentOptionName         = "silent"
+	repoAllowDowngradeOptionName = "allow-downgrade"
 )
 
 var repoGcCmd = &cmds.Command{
@@ -387,3 +392,66 @@ var repoVersionCmd = &cmds.Command{
 		}),
 	},
 }
+
+var repoMigrateCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Apply any outstanding migrations to the repo.",
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption(repoAllowDowngradeOptionName, "Allow downgrading to a lower repo version"),
+	},
+	NoRemote: true,
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		cctx := env.(*oldcmds.Context)
+		allowDowngrade, _ := req.Options[repoAllowDowngradeOptionName].(bool)
+
+		_, err := fsrepo.Open(cctx.ConfigRoot)
+
+		if err == nil {
+			fmt.Println("Repo does not require migration.")
+			return nil
+		} else if err != fsrepo.ErrNeedMigration {
+			return err
+		}
+
+		fmt.Println("Found outdated fs-repo, starting migration.")
+
+		// Read Migration section of IPFS config
+		configFileOpt, _ := req.Options[ConfigFileOption].(string)
+		migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot, configFileOpt)
+		if err != nil {
+			return err
+		}
+
+		// Define function to create IPFS fetcher.  Do not supply an
+		// already-constructed IPFS fetcher, because this may be expensive and
+		// not needed according to migration config. Instead, supply a function
+		// to construct the particular IPFS fetcher implementation used here,
+		// which is called only if an IPFS fetcher is needed.
+		newIpfsFetcher := func(distPath string) migrations.Fetcher {
+			return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot, configFileOpt)
+		}
+
+		// Fetch migrations from current distribution, or location from environ
+		fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)
+
+		// Create fetchers according to migrationCfg.DownloadSources
+		fetcher, err := migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
+		if err != nil {
+			return err
+		}
+		defer fetcher.Close()
+
+		err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", allowDowngrade)
+		if err != nil {
+			fmt.Println("The migrations of fs-repo failed:")
+			fmt.Printf("  %s\n", err)
+			fmt.Println("If you think this is a bug, please file an issue and include this whole log output.")
+			fmt.Println("  https://github.com/ipfs/fs-repo-migrations")
+			return err
+		}
+
+		fmt.Printf("Success: fs-repo has been migrated to version %d.\n", fsrepo.RepoVersion)
+		return nil
+	},
+}

repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go

@@ -247,8 +247,6 @@ func (f *IpfsFetcher) startTempNode(ctx context.Context) error {
 		cancel()
 		// Wait until ipfs is stopped
 		<-node.Context().Done()
-
-		fmt.Println("migration peer", node.Identity, "shutdown")
 	}
 
 	addrs, err := ipfs.Swarm().LocalAddrs(ctx)

test/sharness/t0066-migration.sh

@@ -84,4 +84,42 @@ test_expect_success "output looks good" '
   grep "Please get fs-repo-migrations from https://dist.ipfs.io" daemon_out > /dev/null
 '
 
+test_expect_success "ipfs repo migrate succeed" '
+  test_expect_code 0 ipfs repo migrate > migrate_out
+'
+
+test_expect_success "output looks good" '
+  grep "Found outdated fs-repo, starting migration." migrate_out > /dev/null &&
+  grep "Success: fs-repo migrated to version $IPFS_REPO_VER" true_out > /dev/null
+'
+
+test_expect_success "manually reset repo version to latest" '
+  echo "$IPFS_REPO_VER" > "$IPFS_PATH"/version
+'
+
+test_expect_success "detect repo does not need migration" '
+  test_expect_code 0 ipfs repo migrate > migrate_out
+'
+
+test_expect_success "output looks good" '
+  grep "Repo does not require migration" migrate_out > /dev/null
+'
+
+# ensure that we get a lock error if we need to migrate and the daemon is running
+test_launch_ipfs_daemon
+
+test_expect_success "manually reset repo version to $MIGRATION_START" '
+  echo "$MIGRATION_START" > "$IPFS_PATH"/version
+'
+
+test_expect_success "ipfs repo migrate fails" '
+  test_expect_code 1 ipfs repo migrate 2> migrate_out
+'
+
+test_expect_success "output looks good" '
+  grep "repo.lock" migrate_out > /dev/null
+'
+
+test_kill_ipfs_daemon
+
 test_done