Automatically download and run migrations if needed

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

core/commands/repo.go

@@ -320,7 +320,7 @@ var repoVersionCmd = &cmds.Command{
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		res.SetOutput(&RepoVersion{
-			Version: fsrepo.RepoVersion,
+			Version: fmt.Sprint(fsrepo.RepoVersion),
 		})
 	},
 	Type: RepoVersion{},

core/commands/version.go

@@ -35,7 +35,7 @@ var VersionCmd = &cmds.Command{
 		res.SetOutput(&VersionOutput{
 			Version: config.CurrentVersionNumber,
 			Commit:  config.CurrentCommit,
-			Repo:    fsrepo.RepoVersion,
+			Repo:    fmt.Sprint(fsrepo.RepoVersion),
 			System:  runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here
 			Golang:  runtime.Version(),
 		})

core/corerepo/stat.go

@@ -1,6 +1,8 @@
 package corerepo
 
 import (
+	"fmt"
+
 	"github.com/ipfs/go-ipfs/core"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
@@ -40,6 +42,6 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) {
 		NumObjects: count,
 		RepoSize:   usage,
 		RepoPath:   path,
-		Version:    "fs-repo@" + fsrepo.RepoVersion,
+		Version:    fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion),
 	}, nil
 }

repo/fsrepo/fsrepo.go

@@ -26,7 +26,7 @@ import (
 var log = logging.Logger("fsrepo")
 
 // version number that we are currently expecting to see
-var RepoVersion = "4"
+var RepoVersion = 4
 
 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
 Sorry for the inconvenience. In the future, these will run automatically.`
@@ -36,6 +36,12 @@ Program version is: %s
 Please run the ipfs migration tool before continuing.
 ` + migrationInstructions
 
+var programTooLowMessage = `Your programs version (%d) is lower than your repos (%d).
+Please update ipfs to a version that supports the existing repo, or run
+a migration in reverse.
+
+See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.`
+
 var (
 	ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions)
 	ErrOldRepo   = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions)
@@ -134,8 +140,22 @@ func open(repoPath string) (repo.Repo, error) {
 		return nil, err
 	}
 
-	if ver != RepoVersion {
-		return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion)
+	if RepoVersion > ver {
+		r.lockfile.Close()
+
+		err := mfsr.TryMigrating(RepoVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		r.lockfile, err = lockfile.Lock(r.path)
+		if err != nil {
+			return nil, fmt.Errorf("reacquiring lock: %s", err)
+		}
+
+	} else if ver > RepoVersion {
+		// program version too low for existing repo
+		return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver)
 	}
 
 	// check repo path, then check all constituent parts.

repo/fsrepo/migrations/mfsr.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 )
 
@@ -16,27 +17,26 @@ func (rp RepoPath) VersionFile() string {
 	return path.Join(string(rp), VersionFile)
 }
 
-func (rp RepoPath) Version() (string, error) {
+func (rp RepoPath) Version() (int, error) {
 	if rp == "" {
-		return "", fmt.Errorf("invalid repo path \"%s\"", rp)
+		return 0, fmt.Errorf("invalid repo path \"%s\"", rp)
 	}
 
 	fn := rp.VersionFile()
 	if _, err := os.Stat(fn); os.IsNotExist(err) {
-		return "", VersionFileNotFound(rp)
+		return 0, VersionFileNotFound(rp)
 	}
 
 	c, err := ioutil.ReadFile(fn)
 	if err != nil {
-		return "", err
+		return 0, err
 	}
 
-	s := string(c)
-	s = strings.TrimSpace(s)
-	return s, nil
+	s := strings.TrimSpace(string(c))
+	return strconv.Atoi(s)
 }
 
-func (rp RepoPath) CheckVersion(version string) error {
+func (rp RepoPath) CheckVersion(version int) error {
 	v, err := rp.Version()
 	if err != nil {
 		return err
@@ -49,9 +49,9 @@ func (rp RepoPath) CheckVersion(version string) error {
 	return nil
 }
 
-func (rp RepoPath) WriteVersion(version string) error {
+func (rp RepoPath) WriteVersion(version int) error {
 	fn := rp.VersionFile()
-	return ioutil.WriteFile(fn, []byte(version+"\n"), 0644)
+	return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644)
 }
 
 type VersionFileNotFound string
@@ -59,3 +59,26 @@ type VersionFileNotFound string
 func (v VersionFileNotFound) Error() string {
 	return "no version file in repo at " + string(v)
 }
+
+func TryMigrating(tovers int) error {
+	if !YesNoPrompt("run migrations automatically? [y/n]") {
+		return fmt.Errorf("please run the migrations manually")
+	}
+
+	return RunMigration(tovers)
+}
+
+func YesNoPrompt(prompt string) bool {
+	var s string
+	for {
+		fmt.Printf("%s ", prompt)
+		fmt.Scanf("%s", &s)
+		switch s {
+		case "y", "Y":
+			return true
+		case "n", "N":
+			return false
+		}
+		fmt.Println("Please press either 'y' or 'n'")
+	}
+}

repo/fsrepo/migrations/migrations.go

@@ -0,0 +1,220 @@
+package mfsr
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+var DistPath = "https://ipfs.io/ipns/dist.ipfs.io"
+
+const migrations = "fs-repo-migrations"
+
+func RunMigration(newv int) error {
+	migrateBin := "fs-repo-migrations"
+	fmt.Println("  => checking for migrations binary...")
+	_, err := exec.LookPath(migrateBin)
+	if err == nil {
+		// check to make sure migrations binary supports our target version
+		err = verifyMigrationSupportsVersion(migrateBin, newv)
+	}
+
+	if err != nil {
+		fmt.Println("  => usable migrations not found on system, fetching...")
+		loc, err := GetMigrations()
+		if err != nil {
+			return err
+		}
+
+		err = verifyMigrationSupportsVersion(loc, newv)
+		if err != nil {
+			return fmt.Errorf("could not find migrations binary that supports version %d", newv)
+		}
+
+		migrateBin = loc
+	}
+
+	cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	fmt.Printf("  => running migration: '%s -to %d -y'\n\n", migrateBin, newv)
+
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("migration failed: %s", err)
+	}
+
+	fmt.Println("  => migrations binary completed successfully")
+
+	return nil
+}
+
+func GetMigrations() (string, error) {
+	latest, err := GetLatestVersion(DistPath, migrations)
+	if err != nil {
+		return "", fmt.Errorf("getting latest version of fs-repo-migrations: %s", err)
+	}
+
+	dir, err := ioutil.TempDir("", "go-ipfs-migrate")
+	if err != nil {
+		return "", fmt.Errorf("tempdir: %s", err)
+	}
+
+	out := filepath.Join(dir, migrations)
+
+	err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out)
+	if err != nil {
+		fmt.Printf("  => error getting migrations binary: %s\n", err)
+		fmt.Println("  => could not find or install fs-repo-migrations, please manually install it")
+		return "", fmt.Errorf("failed to find migrations binary")
+	}
+
+	err = os.Chmod(out, 0755)
+	if err != nil {
+		return "", err
+	}
+
+	return out, nil
+}
+
+func verifyMigrationSupportsVersion(fsrbin string, vn int) error {
+	sn, err := migrationsVersion(fsrbin)
+	if err != nil {
+		return err
+	}
+
+	if sn >= vn {
+		return nil
+	}
+
+	return fmt.Errorf("migrations binary doesnt support version %d", vn)
+}
+
+func migrationsVersion(bin string) (int, error) {
+	out, err := exec.Command(bin, "-v").CombinedOutput()
+	if err != nil {
+		return 0, fmt.Errorf("failed to check migrations version: %s", err)
+	}
+
+	vs := strings.Trim(string(out), " \n\t")
+	vn, err := strconv.Atoi(vs)
+	if err != nil {
+		return 0, fmt.Errorf("migrations binary version check did not return a number")
+	}
+
+	return vn, nil
+}
+
+func GetVersions(ipfspath, dist string) ([]string, error) {
+	rc, err := httpFetch(ipfspath + "/" + dist + "/versions")
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+
+	var out []string
+	scan := bufio.NewScanner(rc)
+	for scan.Scan() {
+		out = append(out, scan.Text())
+	}
+
+	return out, nil
+}
+
+func GetLatestVersion(ipfspath, dist string) (string, error) {
+	vs, err := GetVersions(ipfspath, dist)
+	if err != nil {
+		return "", err
+	}
+	var latest string
+	for i := len(vs) - 1; i >= 0; i-- {
+		if !strings.Contains(vs[i], "-dev") {
+			latest = vs[i]
+			break
+		}
+	}
+	if latest == "" {
+		return "", fmt.Errorf("couldnt find a non dev version in the list")
+	}
+	return vs[len(vs)-1], nil
+}
+
+func httpGet(url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("http.NewRequest error: %s", err)
+	}
+
+	req.Header.Set("User-Agent", "go-ipfs")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
+	}
+
+	return resp, nil
+}
+
+func httpFetch(url string) (io.ReadCloser, error) {
+	fmt.Printf("fetching url: %s\n", url)
+	resp, err := httpGet(url)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 {
+		mes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("error reading error body: %s", err)
+		}
+
+		return nil, fmt.Errorf("%s: %s", resp.Status, string(mes))
+	}
+
+	return resp.Body, nil
+}
+
+func GetBinaryForVersion(distname, binnom, root, vers, out string) error {
+	dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate")
+	if err != nil {
+		return err
+	}
+
+	var archive string
+	switch runtime.GOOS {
+	case "windows":
+		archive = "zip"
+	default:
+		archive = "tar.gz"
+	}
+	finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, runtime.GOOS, runtime.GOARCH, archive)
+	distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame)
+
+	data, err := httpFetch(distpath)
+	if err != nil {
+		return err
+	}
+
+	arcpath := filepath.Join(dir, finame)
+	fi, err := os.Create(arcpath)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(fi, data)
+	if err != nil {
+		return err
+	}
+	fi.Close()
+
+	return unpackArchive(distname, binnom, arcpath, out, archive)
+}

repo/fsrepo/migrations/unpack.go

@@ -0,0 +1,101 @@
+package mfsr
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+)
+
+func unpackArchive(dist, binnom, path, out, atype string) error {
+	switch atype {
+	case "zip":
+		return unpackZip(dist, binnom, path, out)
+	case "tar.gz":
+		return unpackTgz(dist, binnom, path, out)
+	default:
+		return fmt.Errorf("unrecognized archive type: %s", atype)
+	}
+}
+
+func unpackTgz(dist, binnom, path, out string) error {
+	fi, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fi.Close()
+
+	gzr, err := gzip.NewReader(fi)
+	if err != nil {
+		return err
+	}
+
+	defer gzr.Close()
+
+	var bin io.Reader
+	tarr := tar.NewReader(gzr)
+
+loop:
+	for {
+		th, err := tarr.Next()
+		switch err {
+		default:
+			return err
+		case io.EOF:
+			break loop
+		case nil:
+			// continue
+		}
+
+		if th.Name == dist+"/"+binnom {
+			bin = tarr
+			break
+		}
+	}
+
+	if bin == nil {
+		return fmt.Errorf("no binary found in downloaded archive")
+	}
+
+	return writeToPath(bin, out)
+}
+
+func writeToPath(rc io.Reader, out string) error {
+	binfi, err := os.Create(out)
+	if err != nil {
+		return fmt.Errorf("error opening tmp bin path '%s': %s", out, err)
+	}
+	defer binfi.Close()
+
+	_, err = io.Copy(binfi, rc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func unpackZip(dist, binnom, path, out string) error {
+	zipr, err := zip.OpenReader(path)
+	if err != nil {
+		return fmt.Errorf("error opening zipreader: %s", err)
+	}
+
+	defer zipr.Close()
+
+	var bin io.ReadCloser
+	for _, fis := range zipr.File {
+		if fis.Name == dist+"/"+binnom+".exe" {
+			rc, err := fis.Open()
+			if err != nil {
+				return fmt.Errorf("error extracting binary from archive: %s", err)
+			}
+
+			bin = rc
+		}
+	}
+
+	return writeToPath(bin, out)
+}