From e37d08f2104cbf5d5c639a791404b31570a27a7d Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 16 Dec 2020 18:47:16 -0800 Subject: [PATCH 01/23] Migration downloads individual migration binaries The code in this PR finds the necessary mirgations, downloads the latest version of them from the distribution site, unpacks the executables, and runs the migrations in order. This code is also used to build the ipfs-update tool and the fs-repo-migrations tool. Note: the fs-repo-migrations tool is only used to run stand-alone migrations now and is not used by either go-ipfs or ipfs-update. Additional utility is provided by this PR, that is not specific to migrations: - Find local ipfs directory - Get current repo version - Check for ipfs daemon availability - Get version information about any distribution on distribution site - Fetch and unpack any binary executable over ipfs or http --- cmd/ipfs/daemon.go | 4 +- go.mod | 1 + go.sum | 2 + repo/fsrepo/fsrepo.go | 6 +- repo/fsrepo/migrations/fetch.go | 268 ++++++++++++++++ repo/fsrepo/migrations/fetch_test.go | 129 ++++++++ repo/fsrepo/migrations/ipfsdir.go | 198 ++++++++++++ repo/fsrepo/migrations/ipfsdir_test.go | 156 +++++++++ repo/fsrepo/migrations/mfsr.go | 55 ---- repo/fsrepo/migrations/mfsr_test.go | 43 --- repo/fsrepo/migrations/migrations.go | 367 +++++++++------------- repo/fsrepo/migrations/migrations_test.go | 93 ++++++ repo/fsrepo/migrations/unpack.go | 94 +++--- repo/fsrepo/migrations/versions.go | 90 ++++++ repo/fsrepo/migrations/versions_test.go | 63 ++++ 15 files changed, 1198 insertions(+), 371 deletions(-) create mode 100644 repo/fsrepo/migrations/fetch.go create mode 100644 repo/fsrepo/migrations/fetch_test.go create mode 100644 repo/fsrepo/migrations/ipfsdir.go create mode 100644 repo/fsrepo/migrations/ipfsdir_test.go delete mode 100644 repo/fsrepo/migrations/mfsr.go delete mode 100644 repo/fsrepo/migrations/mfsr_test.go create mode 100644 repo/fsrepo/migrations/migrations_test.go create mode 100644 repo/fsrepo/migrations/versions.go create mode 100644 repo/fsrepo/migrations/versions_test.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 329a29c4c..698b6c8c4 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -26,7 +26,7 @@ import ( libp2p "github.com/ipfs/go-ipfs/core/node/libp2p" nodeMount "github.com/ipfs/go-ipfs/fuse/node" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" - migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" sockets "github.com/libp2p/go-socket-activation" cmds "github.com/ipfs/go-ipfs-cmds" @@ -288,7 +288,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("fs-repo requires migration") } - err = migrate.RunMigration(fsrepo.RepoVersion) + err = migrations.RunMigration(cctx.Context(), fsrepo.RepoVersion, "") if err != nil { fmt.Println("The migrations of fs-repo failed:") fmt.Printf(" %s\n", err) diff --git a/go.mod b/go.mod index 25e25fcd3..190ed2675 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d github.com/cheggaaa/pb v1.0.29 + github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd/v22 v22.1.0 github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 diff --git a/go.sum b/go.sum index dcb0966c8..2a8ce4f4b 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -1029,6 +1030,7 @@ github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD2 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 011c47a1d..fbe131dc1 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -14,7 +14,7 @@ import ( keystore "github.com/ipfs/go-ipfs/keystore" repo "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo/common" - mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" dir "github.com/ipfs/go-ipfs/thirdparty/dir" ds "github.com/ipfs/go-datastore" @@ -142,7 +142,7 @@ func open(repoPath string) (repo.Repo, error) { }() // Check version, and error out if not matching - ver, err := mfsr.RepoPath(r.path).Version() + ver, err := migrations.RepoVersion(r.path) if err != nil { if os.IsNotExist(err) { return nil, ErrNoVersion @@ -291,7 +291,7 @@ func Init(repoPath string, conf *config.Config) error { return err } - if err := mfsr.RepoPath(repoPath).WriteVersion(RepoVersion); err != nil { + if err := migrations.WriteRepoVersion(repoPath, RepoVersion); err != nil { return err } diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go new file mode 100644 index 000000000..436dd8091 --- /dev/null +++ b/repo/fsrepo/migrations/fetch.go @@ -0,0 +1,268 @@ +package migrations + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "runtime" + "strings" +) + +const ( + // Distribution + gatewayURL = "https://ipfs.io" + ipfsDist = "/ipns/dist.ipfs.io" + // ipfsDist = "/ipfs/QmYRLRDKobvg1AXTGeK5Xk6ntWTsjGiHbyNKhWfz7koGpa" + + // Maximum download size + fetchSizeLimit = 1024 * 1024 * 512 +) + +type limitReadCloser struct { + io.Reader + io.Closer +} + +var ipfsDistPath string + +func init() { + SetIpfsDistPath("") +} + +// SetIpfsDistPath sets the ipfs path to the distribution site. If an empty +// string is given, then the path is set using the IPFS_DIST_PATH environ +// veriable, or the default value if that is not defined. +func SetIpfsDistPath(distPath string) { + if distPath != "" { + ipfsDistPath = distPath + return + } + + if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" { + ipfsDistPath = dist + } else { + ipfsDistPath = ipfsDist + } +} + +// FetchBinary downloads an archive from the distribution site and unpacks it. +// +// The base name of the archive file, inside the distribution directory on +// distribution site, may differ from the distribution name. If it does, then +// specify arcName. +// +// The base name of the binary inside the archive may differ from the base +// archive name. If it does, then specify binName. For example, the following +// is needed because the archive "go-ipfs_v0.7.0_linux-amd64.tar.gz" contains a +// binary named "ipfs" +// +// FetchBinary(ctx, "go-ipfs", "v0.7.0", "go-ipfs", "ipfs", tmpDir) +// +// If out is a directory, then the binary is written to that directory with the +// same name it has inside the archive. Otherwise, the binary file is written +// to the file named by out. +func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) (string, error) { + // If archive base name not specified, then it is same as dist. + if arcName == "" { + arcName = dist + } + // If binary base name is not specified, then it is same as archive base name. + if binName == "" { + binName = arcName + } + + // Name of binary that exists inside archive + binName = ExeName(binName) + + // Return error if file exists or stat fails for reason other than not + // exists. If out is a directory, then write extracted binary to that dir. + fi, err := os.Stat(out) + if !os.IsNotExist(err) { + if err != nil { + return "", err + } + if !fi.IsDir() { + return "", &os.PathError{ + Op: "FetchBinary", + Path: out, + Err: os.ErrExist, + } + } + // out exists and is a directory, so compose final name + out = path.Join(out, binName) + } + + // Create temp directory to store download + tmpDir, err := ioutil.TempDir("", arcName) + if err != nil { + return "", err + } + defer os.RemoveAll(tmpDir) + + atype := "tar.gz" + if runtime.GOOS == "windows" { + atype = "zip" + } + + arcName = makeArchiveName(arcName, ver, atype) + arcIpfsPath := makeIpfsPath(dist, ver, arcName) + + // Create a file to write the archive data to + arcPath := path.Join(tmpDir, arcName) + arcFile, err := os.Create(arcPath) + if err != nil { + return "", err + } + defer arcFile.Close() + + // Open connection to download archive from ipfs path + rc, err := fetch(ctx, arcIpfsPath) + if err != nil { + return "", err + } + defer rc.Close() + + // Write download data + _, err = io.Copy(arcFile, rc) + if err != nil { + return "", err + } + arcFile.Close() + + // Unpack the archive and write binary to out + err = unpackArchive(arcPath, atype, dist, binName, out) + if err != nil { + return "", err + } + + // Set mode of binary to executable + err = os.Chmod(out, 0755) + if err != nil { + return "", err + } + + return out, nil +} + +// fetch attempts to fetch the file at the given ipfs path, first using the +// local ipfs api if available, then using http. Returns io.ReadCloser on +// success, which caller must close. +func fetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) { + // Try fetching via ipfs daemon + rc, err := ipfsFetch(ctx, ipfsPath) + if err == nil { + // Transferred using local ipfs daemon + return rc, nil + } + // Try fetching via HTTP + return httpFetch(ctx, gatewayURL+ipfsPath) +} + +// ipfsFetch attempts to fetch the file at the given ipfs path using the local +// ipfs api. Returns io.ReadCloser on success, which caller must close. +func ipfsFetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) { + sh, _, err := ApiShell("") + if err != nil { + return nil, err + } + + resp, err := sh.Request("cat", ipfsPath).Send(ctx) + if err != nil { + return nil, err + } + if resp.Error != nil { + return nil, resp.Error + } + + return newLimitReadCloser(resp.Output, fetchSizeLimit), nil +} + +// httpFetch attempts to fetch the file at the given URL. Returns +// io.ReadCloser on success, which caller must close. +func httpFetch(ctx context.Context, url string) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest error: %s", err) + } + + req.Header.Set("User-Agent", "go-ipfs") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err) + } + + if resp.StatusCode >= 400 { + defer resp.Body.Close() + mes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading error body: %s", err) + } + return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) + } + + return newLimitReadCloser(resp.Body, fetchSizeLimit), nil +} + +func newLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser { + return limitReadCloser{ + Reader: io.LimitReader(rc, limit), + Closer: rc, + } +} + +// osWithVariant returns the OS name with optional variant. +// Currently returns either runtime.GOOS, or "linux-musl". +func osWithVariant() (string, error) { + if runtime.GOOS != "linux" { + return runtime.GOOS, nil + } + + // ldd outputs the system's kind of libc. + // - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23 + // - on alpine: musl libc (x86_64) + // + // we use the combined stdout+stderr, + // because ldd --version prints differently on different OSes. + // - on standard ubuntu: stdout + // - on alpine: stderr (it probably doesn't know the --version flag) + // + // we suppress non-zero exit codes (see last point about alpine). + out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput() + if err != nil { + return "", err + } + + // now just see if we can find "musl" somewhere in the output + scan := bufio.NewScanner(bytes.NewBuffer(out)) + for scan.Scan() { + if strings.Contains(scan.Text(), "musl") { + return "linux-musl", nil + } + } + + return "linux", nil +} + +// makeArchiveName composes the name of a migration binary archive. +// +// The archive name is in the format: name_version_osv-GOARCH.atype +// Example: ipfs-10-to-11_v1.8.0_darwin-amd64.tar.gz +func makeArchiveName(name, ver, atype string) string { + return fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype) +} + +// makeIpfsPath composes the name ipfs path location to download a migration +// binary from the distribution site. +// +// The ipfs path format: distBaseCID/rootdir/version/name/archive +func makeIpfsPath(dist, ver, arcName string) string { + return fmt.Sprintf("%s/%s/%s/%s", ipfsDistPath, dist, ver, arcName) +} diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go new file mode 100644 index 000000000..f23729e3f --- /dev/null +++ b/repo/fsrepo/migrations/fetch_test.go @@ -0,0 +1,129 @@ +package migrations + +import ( + "bufio" + "context" + "io/ioutil" + "os" + "path" + "strings" + "testing" +) + +func TestHttpFetch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + url := gatewayURL + path.Join(ipfsDistPath, distFSRM, distVersions) + rc, err := httpFetch(ctx, url) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + + var out []string + scan := bufio.NewScanner(rc) + for scan.Scan() { + out = append(out, scan.Text()) + } + err = scan.Err() + if err != nil { + t.Fatal("could not read versions:", err) + } + + if len(out) < 14 { + t.Fatal("do not get all expected data") + } + if out[0] != "v1.0.0" { + t.Fatal("expected v1.0.0 as first line, got", out[0]) + } + + // Check bad URL + url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file") + rc, err = httpFetch(ctx, url) + if err == nil || !strings.Contains(err.Error(), "404") { + t.Fatal("expected error 404") + } +} + +func TestIpfsFetch(t *testing.T) { + _, err := ApiEndpoint("") + if err != nil { + t.Skip("skipped - local ipfs daemon not available") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + url := path.Join(ipfsDistPath, distFSRM, distVersions) + rc, err := ipfsFetch(ctx, url) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + + var out []string + scan := bufio.NewScanner(rc) + for scan.Scan() { + out = append(out, scan.Text()) + } + err = scan.Err() + if err != nil { + t.Fatal("could not read versions:", err) + } + + if len(out) < 14 { + t.Fatal("do not get all expected data") + } + if out[0] != "v1.0.0" { + t.Fatal("expected v1.0.0 as first line, got", out[0]) + } + + // Check bad URL + url = path.Join(ipfsDistPath, distFSRM, "no_such_file") + rc, err = ipfsFetch(ctx, url) + if err == nil || !strings.Contains(err.Error(), "no link") { + t.Fatal("expected 'no link' error, got:", err) + } +} + +func TestFetchBinary(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "fetchtest") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + vers, err := DistVersions(ctx, distFSRM, false) + if err != nil { + t.Fatal(err) + } + t.Log("latest version of", distFSRM, "is", vers[len(vers)-1]) + + bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, distFSRM, tmpDir) + if err != nil { + t.Fatal(err) + } + + fi, err := os.Stat(bin) + if os.IsNotExist(err) { + t.Error("expected file to exist:", bin) + } + + t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) + + bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "go-ipfs", "ipfs", tmpDir) + if err != nil { + t.Fatal(err) + } + + fi, err = os.Stat(bin) + if os.IsNotExist(err) { + t.Error("expected file to exist:", bin) + } + + t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) +} diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go new file mode 100644 index 000000000..c3a809041 --- /dev/null +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -0,0 +1,198 @@ +package migrations + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "strconv" + "strings" + "time" + + api "github.com/ipfs/go-ipfs-api" + "github.com/mitchellh/go-homedir" +) + +const ( + envIpfsPath = "IPFS_PATH" + versionFile = "version" + + // Local IPFS API + apiFile = "api" + shellUpTimeout = 2 * time.Second +) + +var ( + disableDirCache bool + ipfsDirCache string + ipfsDirCacheKey string +) + +// ApiEndpoint reads the api file from the local ipfs install directory and +// returns the address:port read from the file. If the ipfs directory is not +// specified then the default location is used. +func ApiEndpoint(ipfsDir string) (string, error) { + ipfsDir, err := checkIpfsDir(ipfsDir) + if err != nil { + return "", err + } + apiPath := path.Join(ipfsDir, apiFile) + + apiData, err := ioutil.ReadFile(apiPath) + if err != nil { + return "", err + } + + val := strings.TrimSpace(string(apiData)) + parts := strings.Split(val, "/") + if len(parts) != 5 { + return "", fmt.Errorf("incorrectly formatted api string: %q", val) + } + + return parts[2] + ":" + parts[4], nil +} + +// ApiShell creates a new ipfs api shell and checks that it is up. If the shell +// is available, then the shell and ipfs version are returned. +func ApiShell(ipfsDir string) (*api.Shell, string, error) { + apiEp, err := ApiEndpoint("") + if err != nil { + return nil, "", err + } + sh := api.NewShell(apiEp) + sh.SetTimeout(shellUpTimeout) + ver, _, err := sh.Version() + if err != nil { + return nil, "", errors.New("ipfs api shell not up") + } + sh.SetTimeout(0) + return sh, ver, nil +} + +// Returns the path of the default ipfs directory. +func IpfsDir() (string, error) { + return checkIpfsDir("") +} + +// RepoVersion returns the version of the repo in the ipfs directory. If the +// ipfs directory is not specified then the default location is used. +func RepoVersion(ipfsDir string) (int, error) { + ipfsDir, err := checkIpfsDir(ipfsDir) + if err != nil { + return 0, err + } + return repoVersion(ipfsDir) +} + +// WriteRepoVersion writes the specified repo version to the repo located in +// ipfsDir. If ipfsDir is not specified, then the default location is used. +func WriteRepoVersion(ipfsDir string, version int) error { + ipfsDir, err := checkIpfsDir(ipfsDir) + if err != nil { + return err + } + + vFilePath := path.Join(ipfsDir, versionFile) + return ioutil.WriteFile(vFilePath, []byte(fmt.Sprintf("%d\n", version)), 0644) +} + +// CacheIpfsDir enables or disables caching the location of the ipfs directory. +// Enabled by default, this avoids subsequent search for and check of the same +// ipfs directory. Disabling the cache may be useful if the location of the +// ipfs directory is expected to change. +func CacheIpfsDir(enable bool) { + if !enable { + disableDirCache = true + ipfsDirCache = "" + ipfsDirCacheKey = "" + homedir.DisableCache = true + homedir.Reset() + } else { + homedir.DisableCache = false + disableDirCache = false + } +} + +func repoVersion(ipfsDir string) (int, error) { + c, err := ioutil.ReadFile(path.Join(ipfsDir, versionFile)) + if err != nil { + if os.IsNotExist(err) { + // IPFS directory exists without version file, so version 0 + return 0, nil + } + return 0, fmt.Errorf("cannot read repo version file: %s", err) + } + + ver, err := strconv.Atoi(strings.TrimSpace(string(c))) + if err != nil { + return 0, errors.New("invalid data in repo version file") + } + return ver, nil +} + +func checkIpfsDir(dir string) (string, error) { + if dir == ipfsDirCacheKey && ipfsDirCache != "" { + return ipfsDirCache, nil + } + + var ( + err error + found string + ) + if dir == "" { + found, err = findIpfsDir() + if err != nil { + return "", fmt.Errorf("could not find ipfs directory: %s", err) + } + } else { + found, err = homedir.Expand(dir) + if err != nil { + return "", err + } + + _, err = os.Stat(found) + if err != nil { + return "", err + } + } + + if !disableDirCache { + ipfsDirCacheKey = dir + ipfsDirCache = found + } + + return found, nil +} + +func findIpfsDir() (string, error) { + ipfspath := os.Getenv(envIpfsPath) + if ipfspath != "" { + expandedPath, err := homedir.Expand(ipfspath) + if err != nil { + return "", err + } + return expandedPath, nil + } + + home, err := homedir.Dir() + if err != nil { + return "", err + } + if home == "" { + return "", errors.New("could not determine IPFS_PATH, home dir not set") + } + + for _, dir := range []string{".go-ipfs", ".ipfs"} { + defaultDir := path.Join(home, dir) + _, err = os.Stat(defaultDir) + if err == nil { + return defaultDir, nil + } + if !os.IsNotExist(err) { + return "", err + } + } + + return "", err +} diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go new file mode 100644 index 000000000..e4a5fa761 --- /dev/null +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -0,0 +1,156 @@ +package migrations + +import ( + "io/ioutil" + "os" + "path" + "testing" +) + +var ( + fakeHome string + fakeIpfs string +) + +func init() { + CacheIpfsDir(false) +} + +func TestRepoDir(t *testing.T) { + var err error + fakeHome, err = ioutil.TempDir("", "testhome") + if err != nil { + panic(err) + } + defer os.RemoveAll(fakeHome) + + os.Setenv("HOME", fakeHome) + fakeIpfs = path.Join(fakeHome, ".ipfs") + + t.Run("testFindIpfsDir", testFindIpfsDir) + t.Run("testCheckIpfsDir", testCheckIpfsDir) + t.Run("testRepoVersion", testRepoVersion) +} + +func testFindIpfsDir(t *testing.T) { + _, err := findIpfsDir() + if err == nil { + t.Fatal("expected error when no .ipfs directory to find") + } + + err = os.Mkdir(fakeIpfs, os.ModePerm) + if err != nil { + panic(err) + } + + dir, err := findIpfsDir() + if err != nil { + t.Fatal(err) + } + if dir != fakeIpfs { + t.Fatal("wrong ipfs directory:", dir) + } + + os.Setenv("IPFS_PATH", "~/.ipfs") + dir, err = findIpfsDir() + if err != nil { + t.Fatal(err) + } + if dir != fakeIpfs { + t.Fatal("wrong ipfs directory:", dir) + } +} + +func testCheckIpfsDir(t *testing.T) { + _, err := checkIpfsDir("~/no_such_dir") + if err == nil { + t.Fatal("expected error from nonexistent directory") + } + + dir, err := checkIpfsDir("~/.ipfs") + if err != nil { + t.Fatal(err) + } + if dir != fakeIpfs { + t.Fatal("wrong ipfs directory:", dir) + } +} + +func testRepoVersion(t *testing.T) { + ver, err := RepoVersion(fakeIpfs) + if err != nil { + t.Fatal(err) + } + if ver != 0 { + t.Fatal("expected version 0 when no version file") + } + + testVer := 42 + err = WriteRepoVersion(fakeIpfs, testVer) + if err != nil { + t.Fatal(err) + } + + ver, err = RepoVersion(fakeIpfs) + if err != nil { + t.Fatal(err) + } + if ver != testVer { + t.Fatalf("expected version %d, got %d", testVer, ver) + } +} + +func TestApiEndpoint(t *testing.T) { + var err error + fakeHome, err = ioutil.TempDir("", "testhome") + if err != nil { + panic(err) + } + defer os.RemoveAll(fakeHome) + defer os.Unsetenv("HOME") + + os.Setenv("HOME", fakeHome) + fakeIpfs = path.Join(fakeHome, ".ipfs") + + err = os.Mkdir(fakeIpfs, os.ModePerm) + if err != nil { + panic(err) + } + + _, err = ApiEndpoint("") + if err == nil { + t.Fatal("expected error when missing api file") + } + + apiPath := path.Join(fakeIpfs, apiFile) + err = ioutil.WriteFile(apiPath, []byte("bad-data"), 0644) + if err != nil { + panic(err) + } + + _, err = ApiEndpoint("") + if err == nil { + t.Fatal("expected error when bad data") + } + + err = ioutil.WriteFile(apiPath, []byte("/ip4/127.0.0.1/tcp/5001"), 0644) + if err != nil { + panic(err) + } + + val, err := ApiEndpoint("") + if err != nil { + t.Fatal(err) + } + if val != "127.0.0.1:5001" { + t.Fatal("got unexpected value:", val) + } + + val2, err := ApiEndpoint(fakeIpfs) + if err != nil { + t.Fatal(err) + } + if val2 != val { + t.Fatal("expected", val, "got", val2) + } +} diff --git a/repo/fsrepo/migrations/mfsr.go b/repo/fsrepo/migrations/mfsr.go deleted file mode 100644 index c0f3b8b48..000000000 --- a/repo/fsrepo/migrations/mfsr.go +++ /dev/null @@ -1,55 +0,0 @@ -package mfsr - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strconv" - "strings" -) - -const VersionFile = "version" - -type RepoPath string - -func (rp RepoPath) VersionFile() string { - return path.Join(string(rp), VersionFile) -} - -func (rp RepoPath) Version() (int, error) { - if rp == "" { - return 0, fmt.Errorf("invalid repo path \"%s\"", rp) - } - - fn := rp.VersionFile() - if _, err := os.Stat(fn); err != nil { - return 0, err - } - - c, err := ioutil.ReadFile(fn) - if err != nil { - return 0, err - } - - s := strings.TrimSpace(string(c)) - return strconv.Atoi(s) -} - -func (rp RepoPath) CheckVersion(version int) error { - v, err := rp.Version() - if err != nil { - return err - } - - if v != version { - return fmt.Errorf("versions differ (expected: %d, actual:%d)", version, v) - } - - return nil -} - -func (rp RepoPath) WriteVersion(version int) error { - fn := rp.VersionFile() - return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644) -} diff --git a/repo/fsrepo/migrations/mfsr_test.go b/repo/fsrepo/migrations/mfsr_test.go deleted file mode 100644 index c93babff7..000000000 --- a/repo/fsrepo/migrations/mfsr_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package mfsr - -import ( - "io/ioutil" - "os" - "strconv" - "testing" - - "github.com/ipfs/go-ipfs/thirdparty/assert" -) - -func testVersionFile(v string, t *testing.T) (rp RepoPath) { - name, err := ioutil.TempDir("", v) - if err != nil { - t.Fatal(err) - } - rp = RepoPath(name) - return rp -} - -func TestVersion(t *testing.T) { - rp := RepoPath("") - _, err := rp.Version() - assert.Err(err, t, "Should throw an error when path is bad,") - - rp = RepoPath("/path/to/nowhere") - _, err = rp.Version() - if !os.IsNotExist(err) { - t.Fatalf("Should throw an `IsNotExist` error when file doesn't exist: %v", err) - } - - fsrepoV := 5 - - rp = testVersionFile(strconv.Itoa(fsrepoV), t) - _, err = rp.Version() - assert.Err(err, t, "Bad VersionFile") - - assert.Nil(rp.WriteVersion(fsrepoV), t, "Trouble writing version") - - assert.Nil(rp.CheckVersion(fsrepoV), t, "Trouble checking the version") - - assert.Err(rp.CheckVersion(1), t, "Should throw an error for the wrong version.") -} diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 33266dd91..aa46dbc41 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -1,282 +1,203 @@ -package mfsr +package migrations import ( - "bufio" - "bytes" + "context" "fmt" - "io" "io/ioutil" - "net/http" + "log" "os" "os/exec" - "path/filepath" + "path" "runtime" - "strconv" "strings" + "sync" ) -var DistPath = "https://ipfs.io/ipfs/QmYRLRDKobvg1AXTGeK5Xk6ntWTsjGiHbyNKhWfz7koGpa" - -func init() { - if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" { - DistPath = dist - } -} - -const migrations = "fs-repo-migrations" - -func migrationsBinName() string { - switch runtime.GOOS { - case "windows": - return migrations + ".exe" - default: - return migrations - } -} - -func RunMigration(newv int) error { - migrateBin := migrationsBinName() - - fmt.Println(" => Looking for suitable fs-repo-migrations binary.") - - var err error - migrateBin, err = exec.LookPath(migrateBin) - if err == nil { - // check to make sure migrations binary supports our target version - err = verifyMigrationSupportsVersion(migrateBin, newv) - } +const ( + // Migrations distribution + distMigsRoot = "go-ipfs-repo-migrations" + distFSRM = "fs-repo-migrations" +) +// RunMigration finds, downloads, and runs the individual migrations needed to +// migrate the repo from its current version to the target version. +func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error { + ipfsDir, err := checkIpfsDir(ipfsDir) if err != nil { - fmt.Println(" => None found, downloading.") + return err + } + fromVer, err := repoVersion(ipfsDir) + if err != nil { + return fmt.Errorf("could not get repo version: %s", err) + } + if fromVer == targetVer { + // repo already at target version number + return nil + } - loc, err := GetMigrations() + log.Print("Looking for suitable migration binaries.") + + migrations, binPaths, err := findMigrations(ctx, fromVer, targetVer) + if err != nil { + return err + } + + // Download migrations that were not found + if len(binPaths) < len(migrations) { + missing := make([]string, 0, len(migrations)-len(binPaths)) + for _, mig := range migrations { + if _, ok := binPaths[mig]; !ok { + missing = append(missing, mig) + } + } + + log.Print("Need", len(missing), "migrations, downloading.") + + tmpDir, err := ioutil.TempDir("", "migrations") if err != nil { - fmt.Println(" => Failed to download fs-repo-migrations.") return err } + defer os.RemoveAll(tmpDir) - err = verifyMigrationSupportsVersion(loc, newv) + fetched, err := fetchMigrations(ctx, missing, tmpDir) if err != nil { - return fmt.Errorf("no fs-repo-migration binary found for version %d: %s", newv, err) + log.Print("Failed to download migrations.") + return err + } + for i := range missing { + binPaths[missing[i]] = fetched[i] } - - migrateBin = loc } - cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - fmt.Printf(" => Running: %s -to %d -y\n", migrateBin, newv) - - err = cmd.Run() - if err != nil { - fmt.Printf(" => Failed: %s -to %d -y\n", migrateBin, newv) - return fmt.Errorf("migration failed: %s", err) + var revert bool + if fromVer > targetVer { + revert = true } - - fmt.Printf(" => Success: fs-repo has been migrated to version %d.\n", newv) + for _, migration := range migrations { + log.Println("Running migration", migration, "...") + err = runMigration(ctx, binPaths[migration], ipfsDir, revert) + if err != nil { + return fmt.Errorf("migration %s failed: %s", migration, err) + } + } + log.Printf("Success: fs-repo migrated to version %d.\n", targetVer) return nil } -func GetMigrations() (string, error) { - latest, err := GetLatestVersion(DistPath, migrations) +func NeedMigration(target int) (bool, error) { + vnum, err := RepoVersion("") if err != nil { - return "", fmt.Errorf("failed to find latest fs-repo-migrations: %s", err) + return false, fmt.Errorf("could not get repo version: %s", err) } - dir, err := ioutil.TempDir("", "go-ipfs-migrate") - if err != nil { - return "", fmt.Errorf("failed to create fs-repo-migrations tempdir: %s", err) - } - - out := filepath.Join(dir, migrationsBinName()) - - err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out) - if err != nil { - return "", fmt.Errorf("failed to download latest fs-repo-migrations: %s", err) - } - - err = os.Chmod(out, 0755) - if err != nil { - return "", err - } - - return out, nil + return vnum != target, nil } -func verifyMigrationSupportsVersion(fsrbin string, vn int) error { - sn, err := migrationsVersion(fsrbin) - if err != nil { - return err +func ExeName(name string) string { + if runtime.GOOS == "windows" { + return name + ".exe" } - - if sn >= vn { - return nil - } - - return fmt.Errorf("migrations binary doesn't support version %d: %s", vn, fsrbin) + return name } -func migrationsVersion(bin string) (int, error) { - out, err := exec.Command(bin, "-v").CombinedOutput() - if err != nil { - return 0, fmt.Errorf("failed to check migrations version: %s", err) - } - - vs := strings.Trim(string(out), " \n\t") - vn, err := strconv.Atoi(vs) - if err != nil { - return 0, fmt.Errorf("migrations binary version check did not return a number: %s", err) - } - - return vn, nil +func migrationName(from, to int) string { + return fmt.Sprintf("ipfs-%d-to-%d", from, to) } -func GetVersions(ipfspath, dist string) ([]string, error) { - rc, err := httpFetch(ipfspath + "/" + dist + "/versions") - if err != nil { - return nil, err - } - defer rc.Close() - - var out []string - scan := bufio.NewScanner(rc) - for scan.Scan() { - out = append(out, scan.Text()) +// findMigrations returns a list of migrations, ordered from first to last +// migration to apply, and a map of locations migration binaries of any +// migrations that were found. +func findMigrations(ctx context.Context, from, to int) ([]string, map[string]string, error) { + step := 1 + count := to - from + if from > to { + step = -1 + count = from - to } - return out, nil -} + migrations := make([]string, 0, count) + binPaths := make(map[string]string, count) -func GetLatestVersion(ipfspath, dist string) (string, error) { - vs, err := GetVersions(ipfspath, dist) - if err != nil { - return "", err - } - var latest string - for i := len(vs) - 1; i >= 0; i-- { - if !strings.Contains(vs[i], "-dev") { - latest = vs[i] - break + for cur := from; cur != to; cur += step { + if ctx.Err() != nil { + return nil, nil, ctx.Err() } - } - if latest == "" { - return "", fmt.Errorf("couldn't find a non dev version in the list") - } - return vs[len(vs)-1], nil -} - -func httpGet(url string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("http.NewRequest error: %s", err) - } - - req.Header.Set("User-Agent", "go-ipfs") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err) - } - - return resp, nil -} - -func httpFetch(url string) (io.ReadCloser, error) { - resp, err := httpGet(url) - if err != nil { - return nil, err - } - - if resp.StatusCode >= 400 { - mes, err := ioutil.ReadAll(resp.Body) + migName := migrationName(cur, cur+step) + migrations = append(migrations, migName) + bin, err := exec.LookPath(migName) if err != nil { - return nil, fmt.Errorf("error reading error body: %s", err) + continue } - - return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) + binPaths[migName] = bin } - - return resp.Body, nil + return migrations, binPaths, nil } -func GetBinaryForVersion(distname, binnom, root, vers, out string) error { - dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate") - if err != nil { - return err +func runMigration(ctx context.Context, binPath, ipfsDir string, revert bool) error { + pathArg := fmt.Sprintf("-path=%s", ipfsDir) + var cmd *exec.Cmd + if revert { + log.Println(" => Running:", binPath, pathArg, "-verbose=true -revert") + cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true", "-revert") + } else { + log.Println(" => Running:", binPath, pathArg, "-verbose=true") + cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true") } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} - var archive string - switch runtime.GOOS { - case "windows": - archive = "zip" - binnom += ".exe" - default: - archive = "tar.gz" - } +// fetchMigrations downloads the requested migrations, and returns a slice with +// the paths of each binary, in the same order as in needed. +func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]string, error) { osv, err := osWithVariant() if err != nil { - return err + return nil, err } - if osv == "linux-musl" { - return fmt.Errorf("linux-musl not supported, you must build the binary from source for your platform") + return nil, fmt.Errorf("linux-musl not supported, you must build the binary from source for your platform") } - finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, osv, runtime.GOARCH, archive) - distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame) - - data, err := httpFetch(distpath) - if err != nil { - return err + var wg sync.WaitGroup + wg.Add(len(needed)) + bins := make([]string, len(needed)) + // Download and unpack all requested migrations concurrently. + for i, name := range needed { + log.Printf("Downloading migration: %s...", name) + go func(i int, name string) { + defer wg.Done() + distDir := path.Join(distMigsRoot, name) + ver, err := LatestDistVersion(ctx, distDir) + if err != nil { + log.Printf("could not get latest version of migration %s: %s", name, err) + return + } + loc, err := FetchBinary(ctx, distDir, ver, name, name, destDir) + if err != nil { + log.Printf("could not download %s: %s", name, err) + return + } + log.Printf("Downloaded and unpacked migration: %s", loc) + bins[i] = loc + }(i, name) } + wg.Wait() - arcpath := filepath.Join(dir, finame) - fi, err := os.Create(arcpath) - if err != nil { - return err - } - - _, err = io.Copy(fi, data) - if err != nil { - return err - } - fi.Close() - - return unpackArchive(distname, binnom, arcpath, out, archive) -} - -// osWithVariant returns the OS name with optional variant. -// Currently returns either runtime.GOOS, or "linux-musl". -func osWithVariant() (string, error) { - if runtime.GOOS != "linux" { - return runtime.GOOS, nil - } - - // ldd outputs the system's kind of libc. - // - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23 - // - on alpine: musl libc (x86_64) - // - // we use the combined stdout+stderr, - // because ldd --version prints differently on different OSes. - // - on standard ubuntu: stdout - // - on alpine: stderr (it probably doesn't know the --version flag) - // - // we suppress non-zero exit codes (see last point about alpine). - out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput() - if err != nil { - return "", err - } - - // now just see if we can find "musl" somewhere in the output - scan := bufio.NewScanner(bytes.NewBuffer(out)) - for scan.Scan() { - if strings.Contains(scan.Text(), "musl") { - return "linux-musl", nil + var fails []string + for i := range bins { + if bins[i] == "" { + fails = append(fails, needed[i]) } } + if len(fails) != 0 { + err = fmt.Errorf("failed to download migrations: %s", strings.Join(fails, " ")) + if ctx.Err() != nil { + err = fmt.Errorf("%s, %s", ctx.Err(), err) + } + return nil, err + } - return "linux", nil + return bins, nil } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go new file mode 100644 index 000000000..27b458200 --- /dev/null +++ b/repo/fsrepo/migrations/migrations_test.go @@ -0,0 +1,93 @@ +package migrations + +import ( + "context" + "io/ioutil" + "os" + "path" + "testing" +) + +func TestFindMigrations(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "migratetest") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + migs, bins, err := findMigrations(ctx, 0, 5) + if err != nil { + t.Fatal(err) + } + if len(bins) != 0 { + t.Fatal("should not have found migrations") + } + + for i := 1; i < 6; i++ { + createFakeBin(i-1, i, tmpDir) + } + + origPath := os.Getenv("PATH") + os.Setenv("PATH", tmpDir) + defer os.Setenv("PATH", origPath) + + migs, bins, err = findMigrations(ctx, 0, 5) + if err != nil { + t.Fatal(err) + } + if len(migs) != 5 { + t.Fatal("expected 5 migrations") + } + if len(bins) != len(migs) { + t.Fatal("missing", len(migs)-len(bins), "migrations") + } + + os.Remove(bins[migs[2]]) + + migs, bins, err = findMigrations(ctx, 0, 5) + if err != nil { + t.Fatal(err) + } + if len(bins) != len(migs)-1 { + t.Fatal("should be missing one migration bin") + } +} + +func TestFetchMigrations(t *testing.T) { + t.Skip("skip - migrations not available on distribution site yet") + + tmpDir, err := ioutil.TempDir("", "migratetest") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + needed := []string{"ipfs-1-to-2", "ipfs-2-to-3"} + fetched, err := fetchMigrations(ctx, needed, tmpDir) + if err != nil { + t.Fatal(err) + } + + for _, bin := range fetched { + _, err = os.Stat(bin) + if os.IsNotExist(err) { + t.Error("expected file to exist:", bin) + } + } +} + +func createFakeBin(from, to int, tmpDir string) { + migPath := path.Join(tmpDir, ExeName(migrationName(from, to))) + emptyFile, err := os.Create(migPath) + if err != nil { + panic(err) + } + emptyFile.Close() + os.Chmod(migPath, 0755) +} diff --git a/repo/fsrepo/migrations/unpack.go b/repo/fsrepo/migrations/unpack.go index 5b563071f..04c429e68 100644 --- a/repo/fsrepo/migrations/unpack.go +++ b/repo/fsrepo/migrations/unpack.go @@ -1,62 +1,89 @@ -package mfsr +package migrations import ( "archive/tar" "archive/zip" "compress/gzip" + "errors" "fmt" "io" "os" ) -func unpackArchive(dist, binnom, path, out, atype string) error { +func unpackArchive(arcPath, atype, root, name, out string) error { + var err error switch atype { - case "zip": - return unpackZip(dist, binnom, path, out) case "tar.gz": - return unpackTgz(dist, binnom, path, out) + err = unpackTgz(arcPath, root, name, out) + case "zip": + err = unpackZip(arcPath, root, name, out) default: - return fmt.Errorf("unrecognized archive type: %s", atype) + err = fmt.Errorf("unrecognized archive type: %s", atype) } -} - -func unpackTgz(dist, binnom, path, out string) error { - fi, err := os.Open(path) if err != nil { return err } + os.Remove(arcPath) + return nil +} + +func unpackTgz(arcPath, root, name, out string) error { + fi, err := os.Open(arcPath) + if err != nil { + return fmt.Errorf("cannot open archive file: %s", err) + } defer fi.Close() gzr, err := gzip.NewReader(fi) if err != nil { - return err + return fmt.Errorf("error opening gzip reader: %s", err) } - defer gzr.Close() var bin io.Reader tarr := tar.NewReader(gzr) -loop: + lookFor := root + "/" + name for { th, err := tarr.Next() - switch err { - default: - return err - case io.EOF: - break loop - case nil: - // continue + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("cannot read archive: %s", err) } - if th.Name == dist+"/"+binnom { + if th.Name == lookFor { bin = tarr break } } if bin == nil { - return fmt.Errorf("no binary found in downloaded archive") + return errors.New("no binary found in archive") + } + + return writeToPath(bin, out) +} + +func unpackZip(arcPath, root, name, out string) error { + zipr, err := zip.OpenReader(arcPath) + if err != nil { + return fmt.Errorf("error opening zip reader: %s", err) + } + defer zipr.Close() + + lookFor := root + "/" + name + var bin io.ReadCloser + for _, fis := range zipr.File { + if fis.Name == lookFor { + rc, err := fis.Open() + if err != nil { + return fmt.Errorf("error extracting binary from archive: %s", err) + } + + bin = rc + } } return writeToPath(bin, out) @@ -73,26 +100,3 @@ func writeToPath(rc io.Reader, out string) error { return err } - -func unpackZip(dist, binnom, path, out string) error { - zipr, err := zip.OpenReader(path) - if err != nil { - return fmt.Errorf("error opening zipreader: %s", err) - } - - defer zipr.Close() - - var bin io.ReadCloser - for _, fis := range zipr.File { - if fis.Name == dist+"/"+binnom { - rc, err := fis.Open() - if err != nil { - return fmt.Errorf("error extracting binary from archive: %s", err) - } - - bin = rc - } - } - - return writeToPath(bin, out) -} diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go new file mode 100644 index 000000000..7e58381d3 --- /dev/null +++ b/repo/fsrepo/migrations/versions.go @@ -0,0 +1,90 @@ +package migrations + +import ( + "bufio" + "context" + "errors" + "fmt" + "os/exec" + "path" + "sort" + "strconv" + "strings" + + "github.com/coreos/go-semver/semver" +) + +const distVersions = "versions" + +// LatestDistVersion returns the latest version, of the specified distribution, +// that is available on the distribution site. +func LatestDistVersion(ctx context.Context, dist string) (string, error) { + vs, err := DistVersions(ctx, dist, false) + if err != nil { + return "", err + } + + for i := len(vs) - 1; i >= 0; i-- { + ver := vs[i] + if !strings.Contains(ver, "-dev") { + return ver, nil + } + } + return "", errors.New("could not find a non dev version") +} + +// DistVersions returns all versions of the specified distribution, that are +// available on the distriburion site. List is in ascending order, unless +// sortDesc is true. +func DistVersions(ctx context.Context, dist string, sortDesc bool) ([]string, error) { + rc, err := fetch(ctx, path.Join(ipfsDistPath, dist, distVersions)) + if err != nil { + return nil, err + } + defer rc.Close() + + prefix := "v" + var vers []*semver.Version + + scan := bufio.NewScanner(rc) + for scan.Scan() { + ver, err := semver.NewVersion(strings.TrimLeft(scan.Text(), prefix)) + if err != nil { + continue + } + vers = append(vers, ver) + } + err = scan.Err() + if err != nil { + return nil, fmt.Errorf("could not read versions: %s", err) + } + + if sortDesc { + sort.Sort(sort.Reverse(semver.Versions(vers))) + } else { + sort.Sort(semver.Versions(vers)) + } + + out := make([]string, len(vers)) + for i := range vers { + out[i] = prefix + vers[i].String() + } + + return out, nil +} + +// IpfsRepoVersion returns the repo version required by the ipfs daemon +func IpfsRepoVersion(ctx context.Context) (int, error) { + out, err := exec.CommandContext(ctx, "ipfs", "version", "--repo").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("%s: %s", err, string(out)) + } + + verStr := strings.TrimSpace(string(out)) + ver, err := strconv.Atoi(verStr) + if err != nil { + return 0, fmt.Errorf("repo version is not an integer: %s", verStr) + } + + return ver, nil +} diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go new file mode 100644 index 000000000..a97254049 --- /dev/null +++ b/repo/fsrepo/migrations/versions_test.go @@ -0,0 +1,63 @@ +package migrations + +import ( + "context" + "os/exec" + "testing" + + "github.com/coreos/go-semver/semver" +) + +const testDist = "go-ipfs" + +func TestDistVersions(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + vers, err := DistVersions(ctx, testDist, true) + if err != nil { + t.Fatal(err) + } + if len(vers) == 0 { + t.Fatal("no versions of", testDist) + } + t.Log("There are", len(vers), "versions of", testDist) + t.Log("Latest 5 are:", vers[:5]) +} + +func TestLatestDistVersion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + latest, err := LatestDistVersion(ctx, testDist) + if err != nil { + t.Fatal(err) + } + if len(latest) < 6 { + t.Fatal("latest version string too short", latest) + } + _, err = semver.NewVersion(latest[1:]) + if err != nil { + t.Fatal("latest version has invalid format:", latest) + } + t.Log("Latest version of", testDist, "is", latest) +} + +func TestIpfsRepoVersion(t *testing.T) { + _, err := exec.LookPath("ipfs") + if err != nil { + t.Skip("ipfs not available") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ipfsRepoVer, err := IpfsRepoVersion(ctx) + if err != nil { + t.Fatal("Could not get required repo version:", err) + } + if ipfsRepoVer < 1 { + t.Fatal("Invalid repo version") + } + t.Log("IPFS repo version:", ipfsRepoVer) +} From 5a3e567321401851e66afb42e85c063feaa6dfb9 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 11 Jan 2021 20:06:06 -0800 Subject: [PATCH 02/23] Do not put migrations under their own root Since the migrations are not displayed on the dirtributions, there is not need to organize them under their own root to reduce visual clutter. Having each migration follow the same path as all other distributions makes each easier to find in the absence of a link displayed on the distributions web page. It also avoids complicating the distribution deployment scripts and allows each migration distribution to be treated the same as any other distribution. --- repo/fsrepo/migrations/fetch_test.go | 4 ++-- repo/fsrepo/migrations/migrations.go | 4 ++-- repo/fsrepo/migrations/migrations_test.go | 24 ++++++++++++++++++----- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index f23729e3f..0e7770c2c 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -40,7 +40,7 @@ func TestHttpFetch(t *testing.T) { // Check bad URL url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file") - rc, err = httpFetch(ctx, url) + _, err = httpFetch(ctx, url) if err == nil || !strings.Contains(err.Error(), "404") { t.Fatal("expected error 404") } @@ -81,7 +81,7 @@ func TestIpfsFetch(t *testing.T) { // Check bad URL url = path.Join(ipfsDistPath, distFSRM, "no_such_file") - rc, err = ipfsFetch(ctx, url) + _, err = ipfsFetch(ctx, url) if err == nil || !strings.Contains(err.Error(), "no link") { t.Fatal("expected 'no link' error, got:", err) } diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index aa46dbc41..8bccc7887 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -14,8 +14,8 @@ import ( ) const ( - // Migrations distribution - distMigsRoot = "go-ipfs-repo-migrations" + // Migrations subdirectory in distribution. Empty for root (no subdir). + distMigsRoot = "" distFSRM = "fs-repo-migrations" ) diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 27b458200..e57093527 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -3,8 +3,10 @@ package migrations import ( "context" "io/ioutil" + "net/http" "os" "path" + "strings" "testing" ) @@ -22,6 +24,9 @@ func TestFindMigrations(t *testing.T) { if err != nil { t.Fatal(err) } + if len(migs) != 5 { + t.Fatal("expected 5 migrations") + } if len(bins) != 0 { t.Fatal("should not have found migrations") } @@ -57,7 +62,16 @@ func TestFindMigrations(t *testing.T) { } func TestFetchMigrations(t *testing.T) { - t.Skip("skip - migrations not available on distribution site yet") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err := LatestDistVersion(ctx, "ipfs-1-to-2") + if err != nil { + if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) { + t.Skip("skip - migrations not yet available on distribution site") + } + t.Fatal(err) + } tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -65,9 +79,6 @@ func TestFetchMigrations(t *testing.T) { } defer os.RemoveAll(tmpDir) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - needed := []string{"ipfs-1-to-2", "ipfs-2-to-3"} fetched, err := fetchMigrations(ctx, needed, tmpDir) if err != nil { @@ -89,5 +100,8 @@ func createFakeBin(from, to int, tmpDir string) { panic(err) } emptyFile.Close() - os.Chmod(migPath, 0755) + err = os.Chmod(migPath, 0755) + if err != nil { + panic(err) + } } From 01df5bd2118d351a9d18428d9b1b44f3eb0c7950 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 14 Jan 2021 15:43:18 -0800 Subject: [PATCH 03/23] Export IpfsDir() and CheckIpfsDir() - IpfsDir gets the location of the ipfs directory, whether it exists or not - CheckIpfsDir get the location and checks whether it exists. --- repo/fsrepo/migrations/ipfsdir.go | 148 ++++++++-------------- repo/fsrepo/migrations/ipfsdir_test.go | 14 +- repo/fsrepo/migrations/migrations.go | 2 +- repo/fsrepo/migrations/migrations_test.go | 1 + 4 files changed, 61 insertions(+), 104 deletions(-) diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index c3a809041..f371f25ea 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -23,17 +23,15 @@ const ( shellUpTimeout = 2 * time.Second ) -var ( - disableDirCache bool - ipfsDirCache string - ipfsDirCacheKey string -) +func init() { + homedir.DisableCache = true +} // ApiEndpoint reads the api file from the local ipfs install directory and // returns the address:port read from the file. If the ipfs directory is not // specified then the default location is used. func ApiEndpoint(ipfsDir string) (string, error) { - ipfsDir, err := checkIpfsDir(ipfsDir) + ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return "", err } @@ -70,15 +68,60 @@ func ApiShell(ipfsDir string) (*api.Shell, string, error) { return sh, ver, nil } -// Returns the path of the default ipfs directory. -func IpfsDir() (string, error) { - return checkIpfsDir("") +// IpfsDir returns the path of the ipfs directory. If dir specified, then +// returns the expanded version dir. If dir is "", then return the directory +// set by IPFS_PATH, or if IPFS_PATH is not set, then return the default +// location in the home directory. +func IpfsDir(dir string) (string, error) { + var err error + if dir != "" { + dir, err = homedir.Expand(dir) + if err != nil { + return "", err + } + return dir, nil + } + + ipfspath := os.Getenv(envIpfsPath) + if ipfspath != "" { + dir, err := homedir.Expand(ipfspath) + if err != nil { + return "", err + } + return dir, nil + } + + home, err := homedir.Dir() + if err != nil { + return "", err + } + if home == "" { + return "", errors.New("could not determine IPFS_PATH, home dir not set") + } + + return path.Join(home, ".ipfs"), nil +} + +// CheckIpfsDir gets the ipfs directory and checks that the directory exists. +func CheckIpfsDir(dir string) (string, error) { + var err error + dir, err = IpfsDir(dir) + if err != nil { + return "", err + } + + _, err = os.Stat(dir) + if err != nil { + return "", err + } + + return dir, nil } // RepoVersion returns the version of the repo in the ipfs directory. If the // ipfs directory is not specified then the default location is used. func RepoVersion(ipfsDir string) (int, error) { - ipfsDir, err := checkIpfsDir(ipfsDir) + ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return 0, err } @@ -88,7 +131,7 @@ func RepoVersion(ipfsDir string) (int, error) { // WriteRepoVersion writes the specified repo version to the repo located in // ipfsDir. If ipfsDir is not specified, then the default location is used. func WriteRepoVersion(ipfsDir string, version int) error { - ipfsDir, err := checkIpfsDir(ipfsDir) + ipfsDir, err := IpfsDir(ipfsDir) if err != nil { return err } @@ -97,23 +140,6 @@ func WriteRepoVersion(ipfsDir string, version int) error { return ioutil.WriteFile(vFilePath, []byte(fmt.Sprintf("%d\n", version)), 0644) } -// CacheIpfsDir enables or disables caching the location of the ipfs directory. -// Enabled by default, this avoids subsequent search for and check of the same -// ipfs directory. Disabling the cache may be useful if the location of the -// ipfs directory is expected to change. -func CacheIpfsDir(enable bool) { - if !enable { - disableDirCache = true - ipfsDirCache = "" - ipfsDirCacheKey = "" - homedir.DisableCache = true - homedir.Reset() - } else { - homedir.DisableCache = false - disableDirCache = false - } -} - func repoVersion(ipfsDir string) (int, error) { c, err := ioutil.ReadFile(path.Join(ipfsDir, versionFile)) if err != nil { @@ -130,69 +156,3 @@ func repoVersion(ipfsDir string) (int, error) { } return ver, nil } - -func checkIpfsDir(dir string) (string, error) { - if dir == ipfsDirCacheKey && ipfsDirCache != "" { - return ipfsDirCache, nil - } - - var ( - err error - found string - ) - if dir == "" { - found, err = findIpfsDir() - if err != nil { - return "", fmt.Errorf("could not find ipfs directory: %s", err) - } - } else { - found, err = homedir.Expand(dir) - if err != nil { - return "", err - } - - _, err = os.Stat(found) - if err != nil { - return "", err - } - } - - if !disableDirCache { - ipfsDirCacheKey = dir - ipfsDirCache = found - } - - return found, nil -} - -func findIpfsDir() (string, error) { - ipfspath := os.Getenv(envIpfsPath) - if ipfspath != "" { - expandedPath, err := homedir.Expand(ipfspath) - if err != nil { - return "", err - } - return expandedPath, nil - } - - home, err := homedir.Dir() - if err != nil { - return "", err - } - if home == "" { - return "", errors.New("could not determine IPFS_PATH, home dir not set") - } - - for _, dir := range []string{".go-ipfs", ".ipfs"} { - defaultDir := path.Join(home, dir) - _, err = os.Stat(defaultDir) - if err == nil { - return defaultDir, nil - } - if !os.IsNotExist(err) { - return "", err - } - } - - return "", err -} diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index e4a5fa761..6bf37b697 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -12,10 +12,6 @@ var ( fakeIpfs string ) -func init() { - CacheIpfsDir(false) -} - func TestRepoDir(t *testing.T) { var err error fakeHome, err = ioutil.TempDir("", "testhome") @@ -33,7 +29,7 @@ func TestRepoDir(t *testing.T) { } func testFindIpfsDir(t *testing.T) { - _, err := findIpfsDir() + _, err := CheckIpfsDir("") if err == nil { t.Fatal("expected error when no .ipfs directory to find") } @@ -43,7 +39,7 @@ func testFindIpfsDir(t *testing.T) { panic(err) } - dir, err := findIpfsDir() + dir, err := IpfsDir("") if err != nil { t.Fatal(err) } @@ -52,7 +48,7 @@ func testFindIpfsDir(t *testing.T) { } os.Setenv("IPFS_PATH", "~/.ipfs") - dir, err = findIpfsDir() + dir, err = IpfsDir("") if err != nil { t.Fatal(err) } @@ -62,12 +58,12 @@ func testFindIpfsDir(t *testing.T) { } func testCheckIpfsDir(t *testing.T) { - _, err := checkIpfsDir("~/no_such_dir") + _, err := CheckIpfsDir("~/no_such_dir") if err == nil { t.Fatal("expected error from nonexistent directory") } - dir, err := checkIpfsDir("~/.ipfs") + dir, err := CheckIpfsDir("~/.ipfs") if err != nil { t.Fatal(err) } diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 8bccc7887..ddd4419ab 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -22,7 +22,7 @@ const ( // RunMigration finds, downloads, and runs the individual migrations needed to // migrate the repo from its current version to the target version. func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error { - ipfsDir, err := checkIpfsDir(ipfsDir) + ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return err } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index e57093527..58085a2b9 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -65,6 +65,7 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + SetIpfsDistPath("/ipfs/QmdFVsmD668ijuBFJwjyXdP5Sq44a5bPNAq3nnQF77kpyJ") _, err := LatestDistVersion(ctx, "ipfs-1-to-2") if err != nil { if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) { From 048db7599a9a498797a8f685739a880bc9f83e76 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 14 Jan 2021 17:39:26 -0800 Subject: [PATCH 04/23] When version file is missing, do not assume this indicates version 0 since that breaks tests that get the version to see if ipfs is initialized --- repo/fsrepo/migrations/ipfsdir.go | 6 +----- repo/fsrepo/migrations/ipfsdir_test.go | 10 ++++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index f371f25ea..7d374d316 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -143,11 +143,7 @@ func WriteRepoVersion(ipfsDir string, version int) error { func repoVersion(ipfsDir string) (int, error) { c, err := ioutil.ReadFile(path.Join(ipfsDir, versionFile)) if err != nil { - if os.IsNotExist(err) { - // IPFS directory exists without version file, so version 0 - return 0, nil - } - return 0, fmt.Errorf("cannot read repo version file: %s", err) + return 0, err } ver, err := strconv.Atoi(strings.TrimSpace(string(c))) diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index 6bf37b697..c4861ece7 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -73,12 +73,9 @@ func testCheckIpfsDir(t *testing.T) { } func testRepoVersion(t *testing.T) { - ver, err := RepoVersion(fakeIpfs) - if err != nil { - t.Fatal(err) - } - if ver != 0 { - t.Fatal("expected version 0 when no version file") + _, err := RepoVersion(fakeIpfs) + if !os.IsNotExist(err) { + t.Fatal("expected not-exist error") } testVer := 42 @@ -87,6 +84,7 @@ func testRepoVersion(t *testing.T) { t.Fatal(err) } + var ver int ver, err = RepoVersion(fakeIpfs) if err != nil { t.Fatal(err) From afcda7c05b6b37f392390202d13eda00650045cf Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 14 Jan 2021 18:31:37 -0800 Subject: [PATCH 05/23] Fix migraion name when fetching in reverse order --- repo/fsrepo/migrations/migrations.go | 7 +++- repo/fsrepo/migrations/migrations_test.go | 51 +++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index ddd4419ab..1a75962a5 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -123,7 +123,12 @@ func findMigrations(ctx context.Context, from, to int) ([]string, map[string]str if ctx.Err() != nil { return nil, nil, ctx.Err() } - migName := migrationName(cur, cur+step) + var migName string + if step == -1 { + migName = migrationName(cur+step, cur) + } else { + migName = migrationName(cur, cur+step) + } migrations = append(migrations, migName) bin, err := exec.LookPath(migName) if err != nil { diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 58085a2b9..59f9d9864 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -61,6 +61,57 @@ func TestFindMigrations(t *testing.T) { } } +func TestFindMigrationsReverse(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "migratetest") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + migs, bins, err := findMigrations(ctx, 5, 0) + if err != nil { + t.Fatal(err) + } + if len(migs) != 5 { + t.Fatal("expected 5 migrations") + } + if len(bins) != 0 { + t.Fatal("should not have found migrations") + } + + for i := 1; i < 6; i++ { + createFakeBin(i-1, i, tmpDir) + } + + origPath := os.Getenv("PATH") + os.Setenv("PATH", tmpDir) + defer os.Setenv("PATH", origPath) + + migs, bins, err = findMigrations(ctx, 5, 0) + if err != nil { + t.Fatal(err) + } + if len(migs) != 5 { + t.Fatal("expected 5 migrations") + } + if len(bins) != len(migs) { + t.Fatal("missing", len(migs)-len(bins), "migrations:", migs) + } + + os.Remove(bins[migs[2]]) + + migs, bins, err = findMigrations(ctx, 5, 0) + if err != nil { + t.Fatal(err) + } + if len(bins) != len(migs)-1 { + t.Fatal("should be missing one migration bin") + } +} + func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From e205b664a2da1994c06fbc8e4bdf433e75115f7f Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 27 Jan 2021 20:32:43 -0800 Subject: [PATCH 06/23] Fix error reading zip when archive not found - Udate path to IPFS dist - Improve test coverage --- repo/fsrepo/migrations/fetch.go | 4 +- repo/fsrepo/migrations/fetch_test.go | 86 ++++++++- repo/fsrepo/migrations/ipfsdir_test.go | 82 ++++++++- repo/fsrepo/migrations/migrations_test.go | 2 +- repo/fsrepo/migrations/unpack.go | 5 + repo/fsrepo/migrations/unpack_test.go | 212 ++++++++++++++++++++++ 6 files changed, 382 insertions(+), 9 deletions(-) create mode 100644 repo/fsrepo/migrations/unpack_test.go diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 436dd8091..d52c2ef89 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -16,6 +16,8 @@ import ( ) const ( + envIpfsDistPath = "IPFS_DIST_PATH" + // Distribution gatewayURL = "https://ipfs.io" ipfsDist = "/ipns/dist.ipfs.io" @@ -45,7 +47,7 @@ func SetIpfsDistPath(distPath string) { return } - if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" { + if dist := os.Getenv(envIpfsDistPath); dist != "" { ipfsDistPath = dist } else { ipfsDistPath = ipfsDist diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 0e7770c2c..18bb25955 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -10,6 +10,35 @@ import ( "testing" ) +func TestSetIpfsDistPath(t *testing.T) { + os.Unsetenv(envIpfsDistPath) + SetIpfsDistPath("") + if ipfsDistPath != ipfsDist { + t.Error("did not set default dist path") + } + + testDist := "/unit/test/dist" + err := os.Setenv(envIpfsDistPath, testDist) + if err != nil { + panic(err) + } + defer func() { + os.Unsetenv(envIpfsDistPath) + SetIpfsDistPath("") + }() + + SetIpfsDistPath("") + if ipfsDistPath != testDist { + t.Error("did not set dist path from environ") + } + + testDist = "/unit/test/dist2" + SetIpfsDistPath(testDist) + if ipfsDistPath != testDist { + t.Error("did not set dist path") + } +} + func TestHttpFetch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -39,11 +68,24 @@ func TestHttpFetch(t *testing.T) { } // Check bad URL + _, err = httpFetch(ctx, "") + if err == nil { + t.Fatal("expected error") + } + + // Check unreachable URL + _, err = httpFetch(ctx, "http://127.0.0.123:65510") + if err == nil || !strings.HasSuffix(err.Error(), "connection refused") { + t.Fatal("expected 'connection refused' error") + } + + // Check not found url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file") _, err = httpFetch(ctx, url) if err == nil || !strings.Contains(err.Error(), "404") { t.Fatal("expected error 404") } + } func TestIpfsFetch(t *testing.T) { @@ -103,7 +145,7 @@ func TestFetchBinary(t *testing.T) { } t.Log("latest version of", distFSRM, "is", vers[len(vers)-1]) - bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, distFSRM, tmpDir) + bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, "", tmpDir) if err != nil { t.Fatal(err) } @@ -115,7 +157,7 @@ func TestFetchBinary(t *testing.T) { t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) - bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "go-ipfs", "ipfs", tmpDir) + bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir) if err != nil { t.Fatal(err) } @@ -126,4 +168,44 @@ func TestFetchBinary(t *testing.T) { } t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) + + // Check error is destination already exists and is not directory + _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", bin) + if !os.IsExist(err) { + t.Fatal("expected 'exists' error") + } + + // Check error creating temp download directory + err = os.Chmod(tmpDir, 0555) + if err != nil { + panic(err) + } + err = os.Setenv("TMPDIR", tmpDir) + if err != nil { + panic(err) + } + _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir) + if !os.IsPermission(err) { + t.Error("expected 'permission'error") + } + err = os.Setenv("TMPDIR", "/tmp") + if err != nil { + panic(err) + } + err = os.Chmod(tmpDir, 0755) + if err != nil { + panic(err) + } + + // Check error if failure to fetch due to bad dist + _, err = FetchBinary(ctx, "no-such-dist", "v0.3.5", "", "ipfs", tmpDir) + if err == nil || !strings.Contains(err.Error(), "Not Found") { + t.Error("expected 'Not Found' error") + } + + // Check error if failure to unpack archive + _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "not-such-bin", tmpDir) + if err == nil || err.Error() != "no binary found in archive" { + t.Error("expected 'no binary found in archive' error") + } } diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index c4861ece7..59c14a164 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -23,12 +23,12 @@ func TestRepoDir(t *testing.T) { os.Setenv("HOME", fakeHome) fakeIpfs = path.Join(fakeHome, ".ipfs") - t.Run("testFindIpfsDir", testFindIpfsDir) + t.Run("testIpfsDir", testIpfsDir) t.Run("testCheckIpfsDir", testCheckIpfsDir) t.Run("testRepoVersion", testRepoVersion) } -func testFindIpfsDir(t *testing.T) { +func testIpfsDir(t *testing.T) { _, err := CheckIpfsDir("") if err == nil { t.Fatal("expected error when no .ipfs directory to find") @@ -47,7 +47,7 @@ func testFindIpfsDir(t *testing.T) { t.Fatal("wrong ipfs directory:", dir) } - os.Setenv("IPFS_PATH", "~/.ipfs") + os.Setenv(envIpfsPath, "~/.ipfs") dir, err = IpfsDir("") if err != nil { t.Fatal(err) @@ -55,10 +55,46 @@ func testFindIpfsDir(t *testing.T) { if dir != fakeIpfs { t.Fatal("wrong ipfs directory:", dir) } + + _, err = IpfsDir("~somesuer/foo") + if err == nil { + t.Fatal("expected error with user-specific home dir") + } + + err = os.Setenv(envIpfsPath, "~somesuer/foo") + if err != nil { + panic(err) + } + _, err = IpfsDir("~somesuer/foo") + if err == nil { + t.Fatal("expected error with user-specific home dir") + } + err = os.Unsetenv(envIpfsPath) + if err != nil { + panic(err) + } + + dir, err = IpfsDir("~/.ipfs") + if err != nil { + t.Fatal(err) + } + if dir != fakeIpfs { + t.Fatal("wrong ipfs directory:", dir) + } + + _, err = IpfsDir("") + if err != nil { + t.Fatal(err) + } } func testCheckIpfsDir(t *testing.T) { - _, err := CheckIpfsDir("~/no_such_dir") + _, err := CheckIpfsDir("~somesuer/foo") + if err == nil { + t.Fatal("expected error with user-specific home dir") + } + + _, err = CheckIpfsDir("~/no_such_dir") if err == nil { t.Fatal("expected error from nonexistent directory") } @@ -73,7 +109,13 @@ func testCheckIpfsDir(t *testing.T) { } func testRepoVersion(t *testing.T) { - _, err := RepoVersion(fakeIpfs) + badDir := "~somesuer/foo" + _, err := RepoVersion(badDir) + if err == nil { + t.Fatal("expected error with user-specific home dir") + } + + _, err = RepoVersion(fakeIpfs) if !os.IsNotExist(err) { t.Fatal("expected not-exist error") } @@ -92,6 +134,29 @@ func testRepoVersion(t *testing.T) { if ver != testVer { t.Fatalf("expected version %d, got %d", testVer, ver) } + + err = WriteRepoVersion(badDir, testVer) + if err == nil { + t.Fatal("expected error with user-specific home dir") + } + + ipfsDir, err := IpfsDir(fakeIpfs) + if err != nil { + t.Fatal(err) + } + vFilePath := path.Join(ipfsDir, versionFile) + err = ioutil.WriteFile(vFilePath, []byte("bad-version-data\n"), 0644) + if err != nil { + panic(err) + } + _, err = RepoVersion(fakeIpfs) + if err == nil || err.Error() != "invalid data in repo version file" { + t.Fatal("expected 'invalid data' error") + } + err = WriteRepoVersion(fakeIpfs, testVer) + if err != nil { + t.Fatal(err) + } } func TestApiEndpoint(t *testing.T) { @@ -147,4 +212,11 @@ func TestApiEndpoint(t *testing.T) { if val2 != val { t.Fatal("expected", val, "got", val2) } + + _, _, err = ApiShell(fakeIpfs) + if err != nil { + if err.Error() != "ipfs api shell not up" { + t.Fatal("expected 'ipfs api shell not up' error") + } + } } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 59f9d9864..5136132dc 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -116,7 +116,7 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - SetIpfsDistPath("/ipfs/QmdFVsmD668ijuBFJwjyXdP5Sq44a5bPNAq3nnQF77kpyJ") + SetIpfsDistPath("/ipfs/QmXt92hFRuvQgFhgHoaMxC4wLFcvKsCywQPTNmPYCGfEV4") _, err := LatestDistVersion(ctx, "ipfs-1-to-2") if err != nil { if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) { diff --git a/repo/fsrepo/migrations/unpack.go b/repo/fsrepo/migrations/unpack.go index 04c429e68..f08a19885 100644 --- a/repo/fsrepo/migrations/unpack.go +++ b/repo/fsrepo/migrations/unpack.go @@ -83,9 +83,14 @@ func unpackZip(arcPath, root, name, out string) error { } bin = rc + break } } + if bin == nil { + return errors.New("no binary found in archive") + } + return writeToPath(bin, out) } diff --git a/repo/fsrepo/migrations/unpack_test.go b/repo/fsrepo/migrations/unpack_test.go new file mode 100644 index 000000000..cc3ec96a6 --- /dev/null +++ b/repo/fsrepo/migrations/unpack_test.go @@ -0,0 +1,212 @@ +package migrations + +import ( + "archive/tar" + "archive/zip" + "bufio" + "compress/gzip" + "io/ioutil" + "os" + "path" + "strings" + "testing" +) + +func TestUnpackArchive(t *testing.T) { + // Check unrecognized archive type + err := unpackArchive("", "no-arch-type", "", "", "") + if err == nil || err.Error() != "unrecognized archive type: no-arch-type" { + t.Fatal("expected 'unrecognized archive type' error") + } + + // Test cannot open errors + err = unpackArchive("no-archive", "tar.gz", "", "", "") + if err == nil || !strings.HasPrefix(err.Error(), "cannot open archive file") { + t.Fatal("expected 'cannot open' error, got:", err) + } + err = unpackArchive("no-archive", "zip", "", "", "") + if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") { + t.Fatal("expected 'cannot open' error, got:", err) + } +} + +func TestUnpackTgz(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "testunpacktgz") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + badTarGzip := path.Join(tmpDir, "bad.tar.gz") + err = ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644) + if err != nil { + panic(err) + } + err = unpackTgz(badTarGzip, "", "abc", "abc") + if err == nil || !strings.HasPrefix(err.Error(), "error opening gzip reader") { + t.Fatal("expected error opening gzip reader, got:", err) + } + + testTarGzip := path.Join(tmpDir, "test.tar.gz") + testData := "some data" + err = writeTarGzip(testTarGzip, "testroot", "testfile", testData) + if err != nil { + panic(err) + } + + out := path.Join(tmpDir, "out.txt") + + // Test looking for file that is not in archive + err = unpackTgz(testTarGzip, "testroot", "abc", out) + if err == nil || err.Error() != "no binary found in archive" { + t.Fatal("expected 'no binary found in archive' error, got:", err) + } + + // Test that unpack works. + err = unpackTgz(testTarGzip, "testroot", "testfile", out) + if err != nil { + t.Fatal(err) + } + + fi, err := os.Stat(out) + if err != nil { + t.Fatal(err) + } + if fi.Size() != int64(len(testData)) { + t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData)) + } + +} + +func TestUnpackZip(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "testunpackzip") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + badZip := path.Join(tmpDir, "bad.zip") + err = ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644) + if err != nil { + panic(err) + } + err = unpackZip(badZip, "", "abc", "abc") + if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") { + t.Fatal("expected error opening zip reader, got:", err) + } + + testZip := path.Join(tmpDir, "test.zip") + testData := "some data" + err = writeZip(testZip, "testroot", "testfile", testData) + if err != nil { + panic(err) + } + + out := path.Join(tmpDir, "out.txt") + + // Test looking for file that is not in archive + err = unpackZip(testZip, "testroot", "abc", out) + if err == nil || err.Error() != "no binary found in archive" { + t.Fatal("expected 'no binary found in archive' error, got:", err) + } + + // Test that unpack works. + err = unpackZip(testZip, "testroot", "testfile", out) + if err != nil { + t.Fatal(err) + } + + fi, err := os.Stat(out) + if err != nil { + t.Fatal(err) + } + if fi.Size() != int64(len(testData)) { + t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData)) + } +} + +func writeTarGzip(archName, root, fileName, data string) error { + archFile, err := os.Create(archName) + if err != nil { + return err + } + defer archFile.Close() + wr := bufio.NewWriter(archFile) + + // gzip writer writes to buffer + gzw := gzip.NewWriter(wr) + defer gzw.Close() + // tar writer writes to gzip + tw := tar.NewWriter(gzw) + defer tw.Close() + + if fileName != "" { + hdr := &tar.Header{ + Name: path.Join(root, fileName), + Mode: 0600, + Size: int64(len(data)), + } + // Write header + if err = tw.WriteHeader(hdr); err != nil { + return err + } + // Write file body + if _, err := tw.Write([]byte(data)); err != nil { + return err + } + } + + if err = tw.Close(); err != nil { + return err + } + // Close gzip writer; finish writing gzip data to buffer + if err = gzw.Close(); err != nil { + return err + } + // Flush buffered data to file + if err = wr.Flush(); err != nil { + return err + } + // Close tar file + if err = archFile.Close(); err != nil { + return err + } + return nil +} + +func writeZip(archName, root, fileName, data string) error { + archFile, err := os.Create(archName) + if err != nil { + return err + } + defer archFile.Close() + wr := bufio.NewWriter(archFile) + + zw := zip.NewWriter(wr) + defer zw.Close() + + // Write file name + f, err := zw.Create(path.Join(root, fileName)) + if err != nil { + return err + } + // Write file data + _, err = f.Write([]byte(data)) + if err != nil { + return err + } + + // Close zip writer + if err = zw.Close(); err != nil { + return err + } + // Flush buffered data to file + if err = wr.Flush(); err != nil { + return err + } + // Close zip file + if err = archFile.Close(); err != nil { + return err + } + return nil +} From 0942e3be66b03406751e6d38bdac46b47a02b93d Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 27 Jan 2021 21:50:36 -0800 Subject: [PATCH 07/23] rebase and resolve conflicts --- go.mod | 1 + go.sum | 3 +++ repo/fsrepo/migrations/fetch.go | 1 - 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 190ed2675..4525aafa2 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/ipfs/go-filestore v0.0.3 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.6.0 + github.com/ipfs/go-ipfs-api v0.2.0 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.6.0 diff --git a/go.sum b/go.sum index 2a8ce4f4b..9dc230ba8 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= @@ -321,6 +322,8 @@ github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0= github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= +github.com/ipfs/go-ipfs-api v0.2.0 h1:BXRctUU8YOUOQT/jW1s56d9wLa85ntOqK6bptvCKb8c= +github.com/ipfs/go-ipfs-api v0.2.0/go.mod h1:zCTyTl+BuyvUqoSmVb8vjezCJLVTW7G/HBZbCXpTgeM= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index d52c2ef89..1cfc904c9 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -21,7 +21,6 @@ const ( // Distribution gatewayURL = "https://ipfs.io" ipfsDist = "/ipns/dist.ipfs.io" - // ipfsDist = "/ipfs/QmYRLRDKobvg1AXTGeK5Xk6ntWTsjGiHbyNKhWfz7koGpa" // Maximum download size fetchSizeLimit = 1024 * 1024 * 512 From 9d1fdccdca84a13b0c7760bb90c0088973f5a63b Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 1 Feb 2021 16:48:49 -0800 Subject: [PATCH 08/23] test with latest distribution --- repo/fsrepo/migrations/migrations.go | 2 +- repo/fsrepo/migrations/migrations_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 1a75962a5..794413e20 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -184,7 +184,7 @@ func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]st log.Printf("could not download %s: %s", name, err) return } - log.Printf("Downloaded and unpacked migration: %s", loc) + log.Printf("Downloaded and unpacked migration: %s (%s)", loc, ver) bins[i] = loc }(i, name) } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 5136132dc..d86ddbb93 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -10,6 +10,8 @@ import ( "testing" ) +const testIPFSDistPath = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" + func TestFindMigrations(t *testing.T) { tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -116,7 +118,7 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - SetIpfsDistPath("/ipfs/QmXt92hFRuvQgFhgHoaMxC4wLFcvKsCywQPTNmPYCGfEV4") + SetIpfsDistPath(testIPFSDistPath) _, err := LatestDistVersion(ctx, "ipfs-1-to-2") if err != nil { if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) { From 515381da1286b967c2e1566a93f7c51b0699ebe2 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 12 Feb 2021 14:13:36 -0800 Subject: [PATCH 09/23] minor code improvememts --- repo/fsrepo/migrations/ipfsdir.go | 15 +++++---------- repo/fsrepo/migrations/migrations.go | 4 ++-- repo/fsrepo/migrations/versions.go | 5 ++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index 7d374d316..bcc3853b1 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -16,6 +16,7 @@ import ( const ( envIpfsPath = "IPFS_PATH" + defIpfsDir = ".ipfs" versionFile = "version" // Local IPFS API @@ -74,6 +75,9 @@ func ApiShell(ipfsDir string) (*api.Shell, string, error) { // location in the home directory. func IpfsDir(dir string) (string, error) { var err error + if dir == "" { + dir = os.Getenv(envIpfsPath) + } if dir != "" { dir, err = homedir.Expand(dir) if err != nil { @@ -82,15 +86,6 @@ func IpfsDir(dir string) (string, error) { return dir, nil } - ipfspath := os.Getenv(envIpfsPath) - if ipfspath != "" { - dir, err := homedir.Expand(ipfspath) - if err != nil { - return "", err - } - return dir, nil - } - home, err := homedir.Dir() if err != nil { return "", err @@ -99,7 +94,7 @@ func IpfsDir(dir string) (string, error) { return "", errors.New("could not determine IPFS_PATH, home dir not set") } - return path.Join(home, ".ipfs"), nil + return path.Join(home, defIpfsDir), nil } // CheckIpfsDir gets the ipfs directory and checks that the directory exists. diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 794413e20..2fd7c7eca 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -106,7 +106,7 @@ func migrationName(from, to int) string { } // findMigrations returns a list of migrations, ordered from first to last -// migration to apply, and a map of locations migration binaries of any +// migration to apply, and a map of locations of migration binaries of any // migrations that were found. func findMigrations(ctx context.Context, from, to int) ([]string, map[string]string, error) { step := 1 @@ -155,7 +155,7 @@ func runMigration(ctx context.Context, binPath, ipfsDir string, revert bool) err } // fetchMigrations downloads the requested migrations, and returns a slice with -// the paths of each binary, in the same order as in needed. +// the paths of each binary, in the same order specified by needed. func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]string, error) { osv, err := osWithVariant() if err != nil { diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go index 7e58381d3..fd45f2180 100644 --- a/repo/fsrepo/migrations/versions.go +++ b/repo/fsrepo/migrations/versions.go @@ -54,9 +54,8 @@ func DistVersions(ctx context.Context, dist string, sortDesc bool) ([]string, er } vers = append(vers, ver) } - err = scan.Err() - if err != nil { - return nil, fmt.Errorf("could not read versions: %s", err) + if scan.Err() != nil { + return nil, fmt.Errorf("could not read versions: %s", scan.Err()) } if sortDesc { From e22413ab927bfb6252b5bf1a9b84c23c54ad8aaa Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 17 Feb 2021 19:12:09 -0800 Subject: [PATCH 10/23] go-ipfs fetches migrations using specific IPFS path --- cmd/ipfs/daemon.go | 2 ++ repo/fsrepo/migrations/fetch.go | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 698b6c8c4..92374dfd8 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -288,6 +288,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("fs-repo requires migration") } + // Fetch migrations using the current distribution + migrations.SetIpfsDistPath(migrations.CurrentIpfsDist) err = migrations.RunMigration(cctx.Context(), fsrepo.RepoVersion, "") if err != nil { fmt.Println("The migrations of fs-repo failed:") diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 1cfc904c9..0814bd0c6 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -16,6 +16,9 @@ import ( ) const ( + // Current dirstibution to fetch migrations from + CurrentIpfsDist = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" + envIpfsDistPath = "IPFS_DIST_PATH" // Distribution @@ -39,7 +42,7 @@ func init() { // SetIpfsDistPath sets the ipfs path to the distribution site. If an empty // string is given, then the path is set using the IPFS_DIST_PATH environ -// veriable, or the default value if that is not defined. +// veriable, or the default dns link value if that is not defined. func SetIpfsDistPath(distPath string) { if distPath != "" { ipfsDistPath = distPath From fcbe47bc4be7e08239ee0819cf279a7f596b6e29 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 26 Feb 2021 08:41:08 -0800 Subject: [PATCH 11/23] Review changes --- cmd/ipfs/daemon.go | 7 +- repo/fsrepo/migrations/fetch.go | 170 +++++---------------- repo/fsrepo/migrations/fetch_test.go | 174 ++++++++++++---------- repo/fsrepo/migrations/fetcher.go | 96 ++++++++++++ repo/fsrepo/migrations/httpfetcher.go | 91 +++++++++++ repo/fsrepo/migrations/ipfsdir.go | 18 --- repo/fsrepo/migrations/ipfsdir_test.go | 7 - repo/fsrepo/migrations/migrations.go | 15 +- repo/fsrepo/migrations/migrations_test.go | 64 ++++++-- repo/fsrepo/migrations/unpack.go | 12 +- repo/fsrepo/migrations/unpack_test.go | 68 +++++---- repo/fsrepo/migrations/versions.go | 16 +- repo/fsrepo/migrations/versions_test.go | 14 +- 13 files changed, 455 insertions(+), 297 deletions(-) create mode 100644 repo/fsrepo/migrations/fetcher.go create mode 100644 repo/fsrepo/migrations/httpfetcher.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 92374dfd8..c9b05497b 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -288,9 +288,10 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("fs-repo requires migration") } - // Fetch migrations using the current distribution - migrations.SetIpfsDistPath(migrations.CurrentIpfsDist) - err = migrations.RunMigration(cctx.Context(), fsrepo.RepoVersion, "") + fetcher := migrations.NewHttpFetcher() + // Fetch migrations from current distribution, or location from environ + fetcher.SetDistPath(migrations.GetDistPathEnv(migrations.CurrentIpfsDist)) + err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", false) if err != nil { fmt.Println("The migrations of fs-repo failed:") fmt.Printf(" %s\n", err) diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 0814bd0c6..282978fe3 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" "os/exec" "path" @@ -15,68 +14,22 @@ import ( "strings" ) -const ( - // Current dirstibution to fetch migrations from - CurrentIpfsDist = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" - - envIpfsDistPath = "IPFS_DIST_PATH" - - // Distribution - gatewayURL = "https://ipfs.io" - ipfsDist = "/ipns/dist.ipfs.io" - - // Maximum download size - fetchSizeLimit = 1024 * 1024 * 512 -) - -type limitReadCloser struct { - io.Reader - io.Closer -} - -var ipfsDistPath string - -func init() { - SetIpfsDistPath("") -} - -// SetIpfsDistPath sets the ipfs path to the distribution site. If an empty -// string is given, then the path is set using the IPFS_DIST_PATH environ -// veriable, or the default dns link value if that is not defined. -func SetIpfsDistPath(distPath string) { - if distPath != "" { - ipfsDistPath = distPath - return - } - - if dist := os.Getenv(envIpfsDistPath); dist != "" { - ipfsDistPath = dist - } else { - ipfsDistPath = ipfsDist - } -} - // FetchBinary downloads an archive from the distribution site and unpacks it. // -// The base name of the archive file, inside the distribution directory on -// distribution site, may differ from the distribution name. If it does, then -// specify arcName. -// // The base name of the binary inside the archive may differ from the base // archive name. If it does, then specify binName. For example, the following // is needed because the archive "go-ipfs_v0.7.0_linux-amd64.tar.gz" contains a // binary named "ipfs" // -// FetchBinary(ctx, "go-ipfs", "v0.7.0", "go-ipfs", "ipfs", tmpDir) +// FetchBinary(ctx, fetcher, "go-ipfs", "v0.7.0", "ipfs", tmpDir) // // If out is a directory, then the binary is written to that directory with the // same name it has inside the archive. Otherwise, the binary file is written // to the file named by out. -func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) (string, error) { - // If archive base name not specified, then it is same as dist. - if arcName == "" { - arcName = dist - } +func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out string) (string, error) { + // The archive file name is the base of dist to support possible subdir in + // dist, for example: "ipfs-repo-migrations/ipfs-11-to-12" + arcName := path.Base(dist) // If binary base name is not specified, then it is same as archive base name. if binName == "" { binName = arcName @@ -101,6 +54,18 @@ func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) ( } // out exists and is a directory, so compose final name out = path.Join(out, binName) + // Check if the binary already exists in the directory + fi, err = os.Stat(out) + if !os.IsNotExist(err) { + if err != nil { + return "", err + } + return "", &os.PathError{ + Op: "FetchBinary", + Path: out, + Err: os.ErrExist, + } + } } // Create temp directory to store download @@ -115,11 +80,10 @@ func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) ( atype = "zip" } - arcName = makeArchiveName(arcName, ver, atype) - arcIpfsPath := makeIpfsPath(dist, ver, arcName) + arcDistPath, arcFullName := makeArchivePath(dist, arcName, ver, atype) // Create a file to write the archive data to - arcPath := path.Join(tmpDir, arcName) + arcPath := path.Join(tmpDir, arcFullName) arcFile, err := os.Create(arcPath) if err != nil { return "", err @@ -127,7 +91,7 @@ func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) ( defer arcFile.Close() // Open connection to download archive from ipfs path - rc, err := fetch(ctx, arcIpfsPath) + rc, err := fetcher.Fetch(ctx, arcDistPath) if err != nil { return "", err } @@ -155,73 +119,6 @@ func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) ( return out, nil } -// fetch attempts to fetch the file at the given ipfs path, first using the -// local ipfs api if available, then using http. Returns io.ReadCloser on -// success, which caller must close. -func fetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) { - // Try fetching via ipfs daemon - rc, err := ipfsFetch(ctx, ipfsPath) - if err == nil { - // Transferred using local ipfs daemon - return rc, nil - } - // Try fetching via HTTP - return httpFetch(ctx, gatewayURL+ipfsPath) -} - -// ipfsFetch attempts to fetch the file at the given ipfs path using the local -// ipfs api. Returns io.ReadCloser on success, which caller must close. -func ipfsFetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) { - sh, _, err := ApiShell("") - if err != nil { - return nil, err - } - - resp, err := sh.Request("cat", ipfsPath).Send(ctx) - if err != nil { - return nil, err - } - if resp.Error != nil { - return nil, resp.Error - } - - return newLimitReadCloser(resp.Output, fetchSizeLimit), nil -} - -// httpFetch attempts to fetch the file at the given URL. Returns -// io.ReadCloser on success, which caller must close. -func httpFetch(ctx context.Context, url string) (io.ReadCloser, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, fmt.Errorf("http.NewRequest error: %s", err) - } - - req.Header.Set("User-Agent", "go-ipfs") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err) - } - - if resp.StatusCode >= 400 { - defer resp.Body.Close() - mes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error reading error body: %s", err) - } - return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes)) - } - - return newLimitReadCloser(resp.Body, fetchSizeLimit), nil -} - -func newLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser { - return limitReadCloser{ - Reader: io.LimitReader(rc, limit), - Closer: rc, - } -} - // osWithVariant returns the OS name with optional variant. // Currently returns either runtime.GOOS, or "linux-musl". func osWithVariant() (string, error) { @@ -255,18 +152,21 @@ func osWithVariant() (string, error) { return "linux", nil } -// makeArchiveName composes the name of a migration binary archive. +// makeArchivePath composes the path, relative to the distribution site, from which to +// download a binary. The path returned does not contain the distribution site path, +// e.g. "/ipns/dist.ipfs.io/", since that is know to the fetcher. // -// The archive name is in the format: name_version_osv-GOARCH.atype -// Example: ipfs-10-to-11_v1.8.0_darwin-amd64.tar.gz -func makeArchiveName(name, ver, atype string) string { - return fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype) -} - -// makeIpfsPath composes the name ipfs path location to download a migration -// binary from the distribution site. +// Returns the archive path and the base name. // -// The ipfs path format: distBaseCID/rootdir/version/name/archive -func makeIpfsPath(dist, ver, arcName string) string { - return fmt.Sprintf("%s/%s/%s/%s", ipfsDistPath, dist, ver, arcName) +// The ipfs path format is: distribution/version/archiveName +// - distribution is the name of a distribution, such as "go-ipfs" +// - version is the version to fetch, such as "v0.8.0-rc2" +// - archiveName is formatted as name_version_osv-GOARCH.atype, such as +// "go-ipfs_v0.8.0-rc2_linux-amd64.tar.gz" +// +// This would form the path: +// go-ipfs/v0.8.0/go-ipfs_v0.8.0_linux-amd64.tar.gz +func makeArchivePath(dist, name, ver, atype string) (string, string) { + arcName := fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype) + return fmt.Sprintf("%s/%s/%s", dist, ver, arcName), arcName } diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 18bb25955..694e22e2b 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -3,17 +3,63 @@ package migrations import ( "bufio" "context" + "fmt" + "io" "io/ioutil" + "net/http" + "net/http/httptest" "os" "path" "strings" "testing" ) -func TestSetIpfsDistPath(t *testing.T) { +func createTestServer() *httptest.Server { + reqHandler := func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + if strings.Contains(r.URL.Path, "not-here") { + http.NotFound(w, r) + } else if strings.HasSuffix(r.URL.Path, "versions") { + fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n") + } else if strings.HasSuffix(r.URL.Path, ".tar.gz") { + createFakeArchive(r.URL.Path, false, w) + } else if strings.HasSuffix(r.URL.Path, "zip") { + createFakeArchive(r.URL.Path, true, w) + } else { + http.NotFound(w, r) + } + } + return httptest.NewServer(http.HandlerFunc(reqHandler)) +} + +func createFakeArchive(name string, archZip bool, w io.Writer) { + fileName := strings.Split(path.Base(name), "_")[0] + root := path.Base(path.Dir(path.Dir(name))) + + // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive. + if fileName == "go-ipfs" { + fileName = "ipfs" + } + + var err error + if archZip { + err = writeZip(root, fileName, "FAKE DATA", w) + } else { + err = writeTarGzip(root, fileName, "FAKE DATA", w) + } + if err != nil { + panic(err) + } +} + +func TestSetDistPath(t *testing.T) { + f1 := NewHttpFetcher() + f2 := NewHttpFetcher() + mf := NewMultiFetcher(f1, f2) + os.Unsetenv(envIpfsDistPath) - SetIpfsDistPath("") - if ipfsDistPath != ipfsDist { + mf.SetDistPath(GetDistPathEnv("")) + if f1.distPath != IpnsIpfsDist { t.Error("did not set default dist path") } @@ -24,17 +70,30 @@ func TestSetIpfsDistPath(t *testing.T) { } defer func() { os.Unsetenv(envIpfsDistPath) - SetIpfsDistPath("") }() - SetIpfsDistPath("") - if ipfsDistPath != testDist { + mf.SetDistPath(GetDistPathEnv("")) + if f1.distPath != testDist { + t.Error("did not set dist path from environ") + } + if f2.distPath != testDist { + t.Error("did not set dist path from environ") + } + + mf.SetDistPath(GetDistPathEnv("ignored")) + if f1.distPath != testDist { + t.Error("did not set dist path from environ") + } + if f2.distPath != testDist { t.Error("did not set dist path from environ") } testDist = "/unit/test/dist2" - SetIpfsDistPath(testDist) - if ipfsDistPath != testDist { + mf.SetDistPath(testDist) + if f1.distPath != testDist { + t.Error("did not set dist path") + } + if f2.distPath != testDist { t.Error("did not set dist path") } } @@ -43,8 +102,12 @@ func TestHttpFetch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - url := gatewayURL + path.Join(ipfsDistPath, distFSRM, distVersions) - rc, err := httpFetch(ctx, url) + fetcher := NewHttpFetcher() + ts := createTestServer() + defer ts.Close() + fetcher.SetGateway(ts.URL) + + rc, err := fetcher.Fetch(ctx, "/versions") if err != nil { t.Fatal(err) } @@ -60,73 +123,18 @@ func TestHttpFetch(t *testing.T) { t.Fatal("could not read versions:", err) } - if len(out) < 14 { + if len(out) < 6 { t.Fatal("do not get all expected data") } if out[0] != "v1.0.0" { t.Fatal("expected v1.0.0 as first line, got", out[0]) } - // Check bad URL - _, err = httpFetch(ctx, "") - if err == nil { - t.Fatal("expected error") - } - - // Check unreachable URL - _, err = httpFetch(ctx, "http://127.0.0.123:65510") - if err == nil || !strings.HasSuffix(err.Error(), "connection refused") { - t.Fatal("expected 'connection refused' error") - } - // Check not found - url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file") - _, err = httpFetch(ctx, url) + _, err = fetcher.Fetch(ctx, "/no_such_file") if err == nil || !strings.Contains(err.Error(), "404") { t.Fatal("expected error 404") } - -} - -func TestIpfsFetch(t *testing.T) { - _, err := ApiEndpoint("") - if err != nil { - t.Skip("skipped - local ipfs daemon not available") - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - url := path.Join(ipfsDistPath, distFSRM, distVersions) - rc, err := ipfsFetch(ctx, url) - if err != nil { - t.Fatal(err) - } - defer rc.Close() - - var out []string - scan := bufio.NewScanner(rc) - for scan.Scan() { - out = append(out, scan.Text()) - } - err = scan.Err() - if err != nil { - t.Fatal("could not read versions:", err) - } - - if len(out) < 14 { - t.Fatal("do not get all expected data") - } - if out[0] != "v1.0.0" { - t.Fatal("expected v1.0.0 as first line, got", out[0]) - } - - // Check bad URL - url = path.Join(ipfsDistPath, distFSRM, "no_such_file") - _, err = ipfsFetch(ctx, url) - if err == nil || !strings.Contains(err.Error(), "no link") { - t.Fatal("expected 'no link' error, got:", err) - } } func TestFetchBinary(t *testing.T) { @@ -139,13 +147,18 @@ func TestFetchBinary(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - vers, err := DistVersions(ctx, distFSRM, false) + fetcher := NewHttpFetcher() + ts := createTestServer() + defer ts.Close() + fetcher.SetGateway(ts.URL) + + vers, err := DistVersions(ctx, fetcher, distFSRM, false) if err != nil { t.Fatal(err) } t.Log("latest version of", distFSRM, "is", vers[len(vers)-1]) - bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, "", tmpDir) + bin, err := FetchBinary(ctx, fetcher, distFSRM, vers[0], "", tmpDir) if err != nil { t.Fatal(err) } @@ -157,7 +170,7 @@ func TestFetchBinary(t *testing.T) { t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) - bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir) + bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) if err != nil { t.Fatal(err) } @@ -170,11 +183,18 @@ func TestFetchBinary(t *testing.T) { t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name()) // Check error is destination already exists and is not directory - _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", bin) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin) if !os.IsExist(err) { - t.Fatal("expected 'exists' error") + t.Fatal("expected 'exists' error, got", err) } + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + if !os.IsExist(err) { + t.Error("expected 'exists' error, got:", err) + } + + os.Remove(path.Join(tmpDir, "ipfs")) + // Check error creating temp download directory err = os.Chmod(tmpDir, 0555) if err != nil { @@ -184,9 +204,9 @@ func TestFetchBinary(t *testing.T) { if err != nil { panic(err) } - _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) if !os.IsPermission(err) { - t.Error("expected 'permission'error") + t.Error("expected 'permission' error, got:", err) } err = os.Setenv("TMPDIR", "/tmp") if err != nil { @@ -198,13 +218,13 @@ func TestFetchBinary(t *testing.T) { } // Check error if failure to fetch due to bad dist - _, err = FetchBinary(ctx, "no-such-dist", "v0.3.5", "", "ipfs", tmpDir) + _, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir) if err == nil || !strings.Contains(err.Error(), "Not Found") { - t.Error("expected 'Not Found' error") + t.Error("expected 'Not Found' error, got:", err) } // Check error if failure to unpack archive - _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "not-such-bin", tmpDir) + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir) if err == nil || err.Error() != "no binary found in archive" { t.Error("expected 'no binary found in archive' error") } diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go new file mode 100644 index 000000000..6338b75fc --- /dev/null +++ b/repo/fsrepo/migrations/fetcher.go @@ -0,0 +1,96 @@ +package migrations + +import ( + "context" + "io" + "os" + "strings" +) + +const ( + // Current dirstibution to fetch migrations from + CurrentIpfsDist = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" + // Distribution IPNS path. Default for fetchers. + IpnsIpfsDist = "/ipns/dist.ipfs.io" + + // Distribution environ variable + envIpfsDistPath = "IPFS_DIST_PATH" +) + +type Fetcher interface { + // Fetch attempts to fetch the file at the given ipfs path. + // Returns io.ReadCloser on success, which caller must close. + Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) + // SetDistPath sets the path to the distribution site for a Fetcher + SetDistPath(distPath string) +} + +// MultiFetcher holds multiple Fetchers and provides a Fetch that tries each +// until one succeeds. +type MultiFetcher struct { + fetchers []Fetcher +} + +type limitReadCloser struct { + io.Reader + io.Closer +} + +// NewMultiFetcher creates a MultiFetcher with the given Fetchers. The +// Fetchers are tried in order ther passed to this function. +func NewMultiFetcher(f ...Fetcher) Fetcher { + mf := &MultiFetcher{ + fetchers: make([]Fetcher, len(f)), + } + copy(mf.fetchers, f) + return mf +} + +// Fetch attempts to fetch the file at each of its fetchers until one succeeds. +// Returns io.ReadCloser on success, which caller must close. +func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) (rc io.ReadCloser, err error) { + for _, fetcher := range f.fetchers { + rc, err = fetcher.Fetch(ctx, ipfsPath) + if err == nil { + // Transferred using this fetcher + return + } + } + return +} + +// SetDistPath sets the path to the distribution site for all fetchers +func (f *MultiFetcher) SetDistPath(distPath string) { + if !strings.HasPrefix(distPath, "/") { + distPath = "/" + distPath + } + for _, fetcher := range f.fetchers { + fetcher.SetDistPath(distPath) + } +} + +// NewLimitReadCloser returns a new io.ReadCloser with the reader wrappen in a +// io.LimitedReader limited to reading the amount specified. +func NewLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser { + return limitReadCloser{ + Reader: io.LimitReader(rc, limit), + Closer: rc, + } +} + +// GetDistPathEnv returns the IPFS path to the distribution site, using +// the value of environ variable specified by envIpfsDistPath. If the environ +// variable is not set, then returns the provided distPath, and if that is not set +// then returns the IPNS path. +// +// To get the IPFS path of the latest distribution, if not overriddin by the +// environ variable: GetDistPathEnv(CurrentIpfsDist) +func GetDistPathEnv(distPath string) string { + if dist := os.Getenv(envIpfsDistPath); dist != "" { + return dist + } + if distPath == "" { + return IpnsIpfsDist + } + return distPath +} diff --git a/repo/fsrepo/migrations/httpfetcher.go b/repo/fsrepo/migrations/httpfetcher.go new file mode 100644 index 000000000..723747025 --- /dev/null +++ b/repo/fsrepo/migrations/httpfetcher.go @@ -0,0 +1,91 @@ +package migrations + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "strings" +) + +const ( + defaultGatewayURL = "https://ipfs.io" + defaultFetchLimit = 1024 * 1024 * 512 +) + +// HttpFetcher fetches files over HTTP +type HttpFetcher struct { + gateway string + distPath string + limit int64 +} + +var _ Fetcher = (*HttpFetcher)(nil) + +// NewHttpFetcher creates a new HttpFetcher +func NewHttpFetcher() *HttpFetcher { + return &HttpFetcher{ + gateway: defaultGatewayURL, + distPath: IpnsIpfsDist, + limit: defaultFetchLimit, + } +} + +// SetGateway sets the gateway URL +func (f *HttpFetcher) SetGateway(gatewayURL string) error { + gwURL, err := url.Parse(gatewayURL) + if err != nil { + return err + } + f.gateway = gwURL.String() + return nil +} + +// SetDistPath sets the path to the distribution site. +func (f *HttpFetcher) SetDistPath(distPath string) { + if !strings.HasPrefix(distPath, "/") { + distPath = "/" + distPath + } + f.distPath = distPath +} + +// SetFetchLimit sets the download size limit. A value of 0 means no limit. +func (f *HttpFetcher) SetFetchLimit(limit int64) { + f.limit = limit +} + +// Fetch attempts to fetch the file at the given path, from the distribution +// site configured for this HttpFetcher. Returns io.ReadCloser on success, +// which caller must close. +func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) { + gwURL := f.gateway + path.Join(f.distPath, filePath) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, gwURL, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest error: %s", err) + } + + req.Header.Set("User-Agent", "go-ipfs") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err) + } + + if resp.StatusCode >= 400 { + defer resp.Body.Close() + mes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading error body: %s", err) + } + return nil, fmt.Errorf("GET %s error: %s: %s", gwURL, resp.Status, string(mes)) + } + + if f.limit != 0 { + return NewLimitReadCloser(resp.Body, f.limit), nil + } + return resp.Body, nil +} diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index bcc3853b1..cbc407ca2 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -10,7 +10,6 @@ import ( "strings" "time" - api "github.com/ipfs/go-ipfs-api" "github.com/mitchellh/go-homedir" ) @@ -52,23 +51,6 @@ func ApiEndpoint(ipfsDir string) (string, error) { return parts[2] + ":" + parts[4], nil } -// ApiShell creates a new ipfs api shell and checks that it is up. If the shell -// is available, then the shell and ipfs version are returned. -func ApiShell(ipfsDir string) (*api.Shell, string, error) { - apiEp, err := ApiEndpoint("") - if err != nil { - return nil, "", err - } - sh := api.NewShell(apiEp) - sh.SetTimeout(shellUpTimeout) - ver, _, err := sh.Version() - if err != nil { - return nil, "", errors.New("ipfs api shell not up") - } - sh.SetTimeout(0) - return sh, ver, nil -} - // IpfsDir returns the path of the ipfs directory. If dir specified, then // returns the expanded version dir. If dir is "", then return the directory // set by IPFS_PATH, or if IPFS_PATH is not set, then return the default diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index 59c14a164..a1003bc7d 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -212,11 +212,4 @@ func TestApiEndpoint(t *testing.T) { if val2 != val { t.Fatal("expected", val, "got", val2) } - - _, _, err = ApiShell(fakeIpfs) - if err != nil { - if err.Error() != "ipfs api shell not up" { - t.Fatal("expected 'ipfs api shell not up' error") - } - } } diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 2fd7c7eca..1833dff30 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -21,7 +21,7 @@ const ( // RunMigration finds, downloads, and runs the individual migrations needed to // migrate the repo from its current version to the target version. -func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error { +func RunMigration(ctx context.Context, fetcher Fetcher, targetVer int, ipfsDir string, allowDowngrade bool) error { ipfsDir, err := CheckIpfsDir(ipfsDir) if err != nil { return err @@ -34,6 +34,9 @@ func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error { // repo already at target version number return nil } + if fromVer > targetVer && !allowDowngrade { + return fmt.Errorf("downgrade not allowed from %d to %d", fromVer, targetVer) + } log.Print("Looking for suitable migration binaries.") @@ -59,7 +62,7 @@ func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error { } defer os.RemoveAll(tmpDir) - fetched, err := fetchMigrations(ctx, missing, tmpDir) + fetched, err := fetchMigrations(ctx, fetcher, missing, tmpDir) if err != nil { log.Print("Failed to download migrations.") return err @@ -156,7 +159,7 @@ func runMigration(ctx context.Context, binPath, ipfsDir string, revert bool) err // fetchMigrations downloads the requested migrations, and returns a slice with // the paths of each binary, in the same order specified by needed. -func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]string, error) { +func fetchMigrations(ctx context.Context, fetcher Fetcher, needed []string, destDir string) ([]string, error) { osv, err := osWithVariant() if err != nil { return nil, err @@ -173,13 +176,13 @@ func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]st log.Printf("Downloading migration: %s...", name) go func(i int, name string) { defer wg.Done() - distDir := path.Join(distMigsRoot, name) - ver, err := LatestDistVersion(ctx, distDir) + dist := path.Join(distMigsRoot, name) + ver, err := LatestDistVersion(ctx, fetcher, dist, false) if err != nil { log.Printf("could not get latest version of migration %s: %s", name, err) return } - loc, err := FetchBinary(ctx, distDir, ver, name, name, destDir) + loc, err := FetchBinary(ctx, fetcher, dist, ver, name, destDir) if err != nil { log.Printf("could not download %s: %s", name, err) return diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index d86ddbb93..2293d775a 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -3,15 +3,12 @@ package migrations import ( "context" "io/ioutil" - "net/http" "os" "path" "strings" "testing" ) -const testIPFSDistPath = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" - func TestFindMigrations(t *testing.T) { tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -118,14 +115,11 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - SetIpfsDistPath(testIPFSDistPath) - _, err := LatestDistVersion(ctx, "ipfs-1-to-2") - if err != nil { - if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) { - t.Skip("skip - migrations not yet available on distribution site") - } - t.Fatal(err) - } + fetcher := NewHttpFetcher() + fetcher.SetDistPath(CurrentIpfsDist) + ts := createTestServer() + defer ts.Close() + fetcher.SetGateway(ts.URL) tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -134,7 +128,7 @@ func TestFetchMigrations(t *testing.T) { defer os.RemoveAll(tmpDir) needed := []string{"ipfs-1-to-2", "ipfs-2-to-3"} - fetched, err := fetchMigrations(ctx, needed, tmpDir) + fetched, err := fetchMigrations(ctx, fetcher, needed, tmpDir) if err != nil { t.Fatal(err) } @@ -147,6 +141,52 @@ func TestFetchMigrations(t *testing.T) { } } +func TestRunMigrations(t *testing.T) { + var err error + fakeHome, err = ioutil.TempDir("", "testhome") + if err != nil { + panic(err) + } + defer os.RemoveAll(fakeHome) + + os.Setenv("HOME", fakeHome) + fakeIpfs := path.Join(fakeHome, ".ipfs") + + err = os.Mkdir(fakeIpfs, os.ModePerm) + if err != nil { + panic(err) + } + + testVer := 11 + err = WriteRepoVersion(fakeIpfs, testVer) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fetcher := NewHttpFetcher() + fetcher.SetDistPath(CurrentIpfsDist) + ts := createTestServer() + defer ts.Close() + fetcher.SetGateway(ts.URL) + + targetVer := 9 + + err = RunMigration(ctx, fetcher, targetVer, fakeIpfs, false) + if err == nil || !strings.HasPrefix(err.Error(), "downgrade not allowed") { + t.Fatal("expected 'downgrade not alloed' error") + } + + err = RunMigration(ctx, fetcher, targetVer, fakeIpfs, true) + if err != nil { + if !strings.HasPrefix(err.Error(), "migration ipfs-10-to-11 failed") { + t.Fatal(err) + } + } +} + func createFakeBin(from, to int, tmpDir string) { migPath := path.Join(tmpDir, ExeName(migrationName(from, to))) emptyFile, err := os.Create(migPath) diff --git a/repo/fsrepo/migrations/unpack.go b/repo/fsrepo/migrations/unpack.go index f08a19885..485d983cf 100644 --- a/repo/fsrepo/migrations/unpack.go +++ b/repo/fsrepo/migrations/unpack.go @@ -30,13 +30,13 @@ func unpackArchive(arcPath, atype, root, name, out string) error { func unpackTgz(arcPath, root, name, out string) error { fi, err := os.Open(arcPath) if err != nil { - return fmt.Errorf("cannot open archive file: %s", err) + return fmt.Errorf("cannot open archive file: %w", err) } defer fi.Close() gzr, err := gzip.NewReader(fi) if err != nil { - return fmt.Errorf("error opening gzip reader: %s", err) + return fmt.Errorf("error opening gzip reader: %w", err) } defer gzr.Close() @@ -50,7 +50,7 @@ func unpackTgz(arcPath, root, name, out string) error { if err == io.EOF { break } - return fmt.Errorf("cannot read archive: %s", err) + return fmt.Errorf("cannot read archive: %w", err) } if th.Name == lookFor { @@ -69,7 +69,7 @@ func unpackTgz(arcPath, root, name, out string) error { func unpackZip(arcPath, root, name, out string) error { zipr, err := zip.OpenReader(arcPath) if err != nil { - return fmt.Errorf("error opening zip reader: %s", err) + return fmt.Errorf("error opening zip reader: %w", err) } defer zipr.Close() @@ -79,7 +79,7 @@ func unpackZip(arcPath, root, name, out string) error { if fis.Name == lookFor { rc, err := fis.Open() if err != nil { - return fmt.Errorf("error extracting binary from archive: %s", err) + return fmt.Errorf("error extracting binary from archive: %w", err) } bin = rc @@ -97,7 +97,7 @@ func unpackZip(arcPath, root, name, out string) error { func writeToPath(rc io.Reader, out string) error { binfi, err := os.Create(out) if err != nil { - return fmt.Errorf("error opening tmp bin path '%s': %s", out, err) + return fmt.Errorf("error creating output file '%s': %w", out, err) } defer binfi.Close() diff --git a/repo/fsrepo/migrations/unpack_test.go b/repo/fsrepo/migrations/unpack_test.go index cc3ec96a6..e772f00b1 100644 --- a/repo/fsrepo/migrations/unpack_test.go +++ b/repo/fsrepo/migrations/unpack_test.go @@ -5,6 +5,7 @@ import ( "archive/zip" "bufio" "compress/gzip" + "io" "io/ioutil" "os" "path" @@ -49,7 +50,7 @@ func TestUnpackTgz(t *testing.T) { testTarGzip := path.Join(tmpDir, "test.tar.gz") testData := "some data" - err = writeTarGzip(testTarGzip, "testroot", "testfile", testData) + err = writeTarGzipFile(testTarGzip, "testroot", "testfile", testData) if err != nil { panic(err) } @@ -97,7 +98,7 @@ func TestUnpackZip(t *testing.T) { testZip := path.Join(tmpDir, "test.zip") testData := "some data" - err = writeZip(testZip, "testroot", "testfile", testData) + err = writeZipFile(testZip, "testroot", "testfile", testData) if err != nil { panic(err) } @@ -125,21 +126,38 @@ func TestUnpackZip(t *testing.T) { } } -func writeTarGzip(archName, root, fileName, data string) error { +func writeTarGzipFile(archName, root, fileName, data string) error { archFile, err := os.Create(archName) if err != nil { return err } defer archFile.Close() - wr := bufio.NewWriter(archFile) + w := bufio.NewWriter(archFile) + err = writeTarGzip(root, fileName, data, w) + if err != nil { + return err + } + // Flush buffered data to file + if err = w.Flush(); err != nil { + return err + } + // Close tar file + if err = archFile.Close(); err != nil { + return err + } + return nil +} + +func writeTarGzip(root, fileName, data string, w io.Writer) error { // gzip writer writes to buffer - gzw := gzip.NewWriter(wr) + gzw := gzip.NewWriter(w) defer gzw.Close() // tar writer writes to gzip tw := tar.NewWriter(gzw) defer tw.Close() + var err error if fileName != "" { hdr := &tar.Header{ Name: path.Join(root, fileName), @@ -163,26 +181,34 @@ func writeTarGzip(archName, root, fileName, data string) error { if err = gzw.Close(); err != nil { return err } - // Flush buffered data to file - if err = wr.Flush(); err != nil { + return nil +} + +func writeZipFile(archName, root, fileName, data string) error { + archFile, err := os.Create(archName) + if err != nil { return err } - // Close tar file + defer archFile.Close() + w := bufio.NewWriter(archFile) + + err = writeZip(root, fileName, data, w) + if err != nil { + return err + } + // Flush buffered data to file + if err = w.Flush(); err != nil { + return err + } + // Close zip file if err = archFile.Close(); err != nil { return err } return nil } -func writeZip(archName, root, fileName, data string) error { - archFile, err := os.Create(archName) - if err != nil { - return err - } - defer archFile.Close() - wr := bufio.NewWriter(archFile) - - zw := zip.NewWriter(wr) +func writeZip(root, fileName, data string, w io.Writer) error { + zw := zip.NewWriter(w) defer zw.Close() // Write file name @@ -200,13 +226,5 @@ func writeZip(archName, root, fileName, data string) error { if err = zw.Close(); err != nil { return err } - // Flush buffered data to file - if err = wr.Flush(); err != nil { - return err - } - // Close zip file - if err = archFile.Close(); err != nil { - return err - } return nil } diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go index fd45f2180..7d9fda835 100644 --- a/repo/fsrepo/migrations/versions.go +++ b/repo/fsrepo/migrations/versions.go @@ -18,17 +18,21 @@ const distVersions = "versions" // LatestDistVersion returns the latest version, of the specified distribution, // that is available on the distribution site. -func LatestDistVersion(ctx context.Context, dist string) (string, error) { - vs, err := DistVersions(ctx, dist, false) +func LatestDistVersion(ctx context.Context, fetcher Fetcher, dist string, stableOnly bool) (string, error) { + vs, err := DistVersions(ctx, fetcher, dist, false) if err != nil { return "", err } for i := len(vs) - 1; i >= 0; i-- { ver := vs[i] - if !strings.Contains(ver, "-dev") { - return ver, nil + if stableOnly && strings.Contains(ver, "-rc") { + continue } + if strings.Contains(ver, "-dev") { + continue + } + return ver, nil } return "", errors.New("could not find a non dev version") } @@ -36,8 +40,8 @@ func LatestDistVersion(ctx context.Context, dist string) (string, error) { // DistVersions returns all versions of the specified distribution, that are // available on the distriburion site. List is in ascending order, unless // sortDesc is true. -func DistVersions(ctx context.Context, dist string, sortDesc bool) ([]string, error) { - rc, err := fetch(ctx, path.Join(ipfsDistPath, dist, distVersions)) +func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bool) ([]string, error) { + rc, err := fetcher.Fetch(ctx, path.Join(dist, distVersions)) if err != nil { return nil, err } diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index a97254049..af1b11505 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -14,7 +14,12 @@ func TestDistVersions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - vers, err := DistVersions(ctx, testDist, true) + fetcher := NewHttpFetcher() + ts := createTestServer() + defer ts.Close() + fetcher.SetGateway(ts.URL) + + vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { t.Fatal(err) } @@ -29,7 +34,12 @@ func TestLatestDistVersion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - latest, err := LatestDistVersion(ctx, testDist) + fetcher := NewHttpFetcher() + //ts := createTestServer() + //defer ts.Close() + //fetcher.SetGateway(ts.URL) + + latest, err := LatestDistVersion(ctx, fetcher, testDist, false) if err != nil { t.Fatal(err) } From 852dfab997f58d9c36765c2fae7e619390f44fdb Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 26 Feb 2021 11:28:00 -0800 Subject: [PATCH 12/23] fix lint warnings --- go.mod | 1 - go.sum | 14 -------------- repo/fsrepo/migrations/fetch.go | 2 +- repo/fsrepo/migrations/fetch_test.go | 9 +++++++-- repo/fsrepo/migrations/ipfsdir.go | 4 +--- repo/fsrepo/migrations/migrations_test.go | 9 +++++++-- repo/fsrepo/migrations/versions_test.go | 5 ++++- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 4525aafa2..190ed2675 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,6 @@ require ( github.com/ipfs/go-filestore v0.0.3 github.com/ipfs/go-fs-lock v0.0.6 github.com/ipfs/go-graphsync v0.6.0 - github.com/ipfs/go-ipfs-api v0.2.0 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-cmds v0.6.0 diff --git a/go.sum b/go.sum index 9dc230ba8..d1141d651 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= -github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -169,7 +167,6 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -322,8 +319,6 @@ github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0= github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk= -github.com/ipfs/go-ipfs-api v0.2.0 h1:BXRctUU8YOUOQT/jW1s56d9wLa85ntOqK6bptvCKb8c= -github.com/ipfs/go-ipfs-api v0.2.0/go.mod h1:zCTyTl+BuyvUqoSmVb8vjezCJLVTW7G/HBZbCXpTgeM= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= @@ -461,7 +456,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -623,7 +617,6 @@ github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6n github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= -github.com/libp2p/go-libp2p-pubsub v0.4.0 h1:YNVRyXqBgv9i4RG88jzoTtkSOaSB45CqHkL29NNBZb4= github.com/libp2p/go-libp2p-pubsub v0.4.0/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ= github.com/libp2p/go-libp2p-pubsub v0.4.1 h1:j4umIg5nyus+sqNfU+FWvb9aeYFQH/A+nDFhWj+8yy8= github.com/libp2p/go-libp2p-pubsub v0.4.1/go.mod h1:izkeMLvz6Ht8yAISXjx60XUQZMq9ZMe5h2ih4dLIBIQ= @@ -994,7 +987,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -1033,7 +1025,6 @@ github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD2 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= -github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= @@ -1161,7 +1152,6 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1181,7 +1171,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1228,7 +1217,6 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1278,13 +1266,11 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 h1:r3P/5xOq/dK1991B65Oy6E1fRF/2d/fSYZJ/fXGVfJc= golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 282978fe3..753c9b10b 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -55,7 +55,7 @@ func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out s // out exists and is a directory, so compose final name out = path.Join(out, binName) // Check if the binary already exists in the directory - fi, err = os.Stat(out) + _, err = os.Stat(out) if !os.IsNotExist(err) { if err != nil { return "", err diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 694e22e2b..c5fa9625f 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -105,7 +105,10 @@ func TestHttpFetch(t *testing.T) { fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - fetcher.SetGateway(ts.URL) + err := fetcher.SetGateway(ts.URL) + if err != nil { + panic(err) + } rc, err := fetcher.Fetch(ctx, "/versions") if err != nil { @@ -150,7 +153,9 @@ func TestFetchBinary(t *testing.T) { fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - fetcher.SetGateway(ts.URL) + if err = fetcher.SetGateway(ts.URL); err != nil { + panic(err) + } vers, err := DistVersions(ctx, fetcher, distFSRM, false) if err != nil { diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index cbc407ca2..9f107dbc2 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -8,7 +8,6 @@ import ( "path" "strconv" "strings" - "time" "github.com/mitchellh/go-homedir" ) @@ -19,8 +18,7 @@ const ( versionFile = "version" // Local IPFS API - apiFile = "api" - shellUpTimeout = 2 * time.Second + apiFile = "api" ) func init() { diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 2293d775a..51e476100 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -119,7 +119,10 @@ func TestFetchMigrations(t *testing.T) { fetcher.SetDistPath(CurrentIpfsDist) ts := createTestServer() defer ts.Close() - fetcher.SetGateway(ts.URL) + err := fetcher.SetGateway(ts.URL) + if err != nil { + panic(err) + } tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -170,7 +173,9 @@ func TestRunMigrations(t *testing.T) { fetcher.SetDistPath(CurrentIpfsDist) ts := createTestServer() defer ts.Close() - fetcher.SetGateway(ts.URL) + if err = fetcher.SetGateway(ts.URL); err != nil { + panic(err) + } targetVer := 9 diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index af1b11505..984d70abd 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -17,7 +17,10 @@ func TestDistVersions(t *testing.T) { fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - fetcher.SetGateway(ts.URL) + err := fetcher.SetGateway(ts.URL) + if err != nil { + panic(err) + } vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { From 077266d3bd1912be5f6335c94ea69ecae25aad04 Mon Sep 17 00:00:00 2001 From: gammazero Date: Mon, 15 Mar 2021 17:13:45 -0700 Subject: [PATCH 13/23] Review changes --- cmd/ipfs/daemon.go | 3 +- repo/fsrepo/migrations/fetch_test.go | 72 +++++++++++++---------- repo/fsrepo/migrations/fetcher.go | 13 ---- repo/fsrepo/migrations/httpfetcher.go | 55 +++++++++-------- repo/fsrepo/migrations/ipfsdir.go | 27 --------- repo/fsrepo/migrations/ipfsdir_test.go | 55 ----------------- repo/fsrepo/migrations/migrations_test.go | 19 ++---- repo/fsrepo/migrations/versions.go | 18 ------ repo/fsrepo/migrations/versions_test.go | 33 ++--------- 9 files changed, 79 insertions(+), 216 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index c9b05497b..874c367c2 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -288,9 +288,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("fs-repo requires migration") } - fetcher := migrations.NewHttpFetcher() // Fetch migrations from current distribution, or location from environ - fetcher.SetDistPath(migrations.GetDistPathEnv(migrations.CurrentIpfsDist)) + fetcher := migrations.NewHttpFetcher(migrations.GetDistPathEnv(migrations.CurrentIpfsDist), "", "go-ipfs", 0) err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", false) if err != nil { fmt.Println("The migrations of fs-repo failed:") diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index c5fa9625f..7441e248e 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -52,14 +52,10 @@ func createFakeArchive(name string, archZip bool, w io.Writer) { } } -func TestSetDistPath(t *testing.T) { - f1 := NewHttpFetcher() - f2 := NewHttpFetcher() - mf := NewMultiFetcher(f1, f2) - +func TestGetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) - mf.SetDistPath(GetDistPathEnv("")) - if f1.distPath != IpnsIpfsDist { + distPath := GetDistPathEnv("") + if distPath != IpnsIpfsDist { t.Error("did not set default dist path") } @@ -72,28 +68,18 @@ func TestSetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) }() - mf.SetDistPath(GetDistPathEnv("")) - if f1.distPath != testDist { + distPath = GetDistPathEnv("") + if distPath != testDist { t.Error("did not set dist path from environ") } - if f2.distPath != testDist { - t.Error("did not set dist path from environ") - } - - mf.SetDistPath(GetDistPathEnv("ignored")) - if f1.distPath != testDist { - t.Error("did not set dist path from environ") - } - if f2.distPath != testDist { + distPath = GetDistPathEnv("ignored") + if distPath != testDist { t.Error("did not set dist path from environ") } testDist = "/unit/test/dist2" - mf.SetDistPath(testDist) - if f1.distPath != testDist { - t.Error("did not set dist path") - } - if f2.distPath != testDist { + fetcher := NewHttpFetcher(testDist, "", "", 0) + if fetcher.distPath != testDist { t.Error("did not set dist path") } } @@ -102,13 +88,10 @@ func TestHttpFetch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - err := fetcher.SetGateway(ts.URL) - if err != nil { - panic(err) - } + + fetcher := NewHttpFetcher("", ts.URL, "", 0) rc, err := fetcher.Fetch(ctx, "/versions") if err != nil { @@ -150,12 +133,10 @@ func TestFetchBinary(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - if err = fetcher.SetGateway(ts.URL); err != nil { - panic(err) - } + + fetcher := NewHttpFetcher("", ts.URL, "", 0) vers, err := DistVersions(ctx, fetcher, distFSRM, false) if err != nil { @@ -234,3 +215,30 @@ func TestFetchBinary(t *testing.T) { t.Error("expected 'no binary found in archive' error") } } + +func TestMultiFetcher(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ts := createTestServer() + defer ts.Close() + + badFetcher := NewHttpFetcher("", "bad-url", "", 0) + fetcher := NewHttpFetcher("", ts.URL, "", 0) + + mf := NewMultiFetcher(badFetcher, fetcher) + + rc, err := mf.Fetch(ctx, "/versions") + if err != nil { + t.Fatal(err) + } + defer rc.Close() + + vers, err := ioutil.ReadAll(rc) + if err != nil { + t.Fatal("could not read versions:", err) + } + if len(vers) < 45 { + fmt.Println("unexpected more data") + } +} diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 6338b75fc..23a2370ce 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -4,7 +4,6 @@ import ( "context" "io" "os" - "strings" ) const ( @@ -21,8 +20,6 @@ type Fetcher interface { // Fetch attempts to fetch the file at the given ipfs path. // Returns io.ReadCloser on success, which caller must close. Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) - // SetDistPath sets the path to the distribution site for a Fetcher - SetDistPath(distPath string) } // MultiFetcher holds multiple Fetchers and provides a Fetch that tries each @@ -59,16 +56,6 @@ func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) (rc io.ReadCl return } -// SetDistPath sets the path to the distribution site for all fetchers -func (f *MultiFetcher) SetDistPath(distPath string) { - if !strings.HasPrefix(distPath, "/") { - distPath = "/" + distPath - } - for _, fetcher := range f.fetchers { - fetcher.SetDistPath(distPath) - } -} - // NewLimitReadCloser returns a new io.ReadCloser with the reader wrappen in a // io.LimitedReader limited to reading the amount specified. func NewLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser { diff --git a/repo/fsrepo/migrations/httpfetcher.go b/repo/fsrepo/migrations/httpfetcher.go index 723747025..efc43bef6 100644 --- a/repo/fsrepo/migrations/httpfetcher.go +++ b/repo/fsrepo/migrations/httpfetcher.go @@ -6,7 +6,6 @@ import ( "io" "io/ioutil" "net/http" - "net/url" "path" "strings" ) @@ -18,43 +17,45 @@ const ( // HttpFetcher fetches files over HTTP type HttpFetcher struct { - gateway string - distPath string - limit int64 + distPath string + gateway string + limit int64 + userAgent string } var _ Fetcher = (*HttpFetcher)(nil) // NewHttpFetcher creates a new HttpFetcher -func NewHttpFetcher() *HttpFetcher { - return &HttpFetcher{ - gateway: defaultGatewayURL, +// +// Specifying "" for distPath sets the default IPNS path. +// Specifying "" for gateway sets the default. +// Specifying 0 for fetchLimit sets the default, -1 means no limit. +func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *HttpFetcher { + f := &HttpFetcher{ distPath: IpnsIpfsDist, + gateway: defaultGatewayURL, limit: defaultFetchLimit, } -} -// SetGateway sets the gateway URL -func (f *HttpFetcher) SetGateway(gatewayURL string) error { - gwURL, err := url.Parse(gatewayURL) - if err != nil { - return err + if distPath != "" { + if !strings.HasPrefix(distPath, "/") { + distPath = "/" + distPath + } + f.distPath = distPath } - f.gateway = gwURL.String() - return nil -} -// SetDistPath sets the path to the distribution site. -func (f *HttpFetcher) SetDistPath(distPath string) { - if !strings.HasPrefix(distPath, "/") { - distPath = "/" + distPath + if gateway != "" { + f.gateway = strings.TrimRight(gateway, "/") } - f.distPath = distPath -} -// SetFetchLimit sets the download size limit. A value of 0 means no limit. -func (f *HttpFetcher) SetFetchLimit(limit int64) { - f.limit = limit + if fetchLimit != 0 { + if fetchLimit == -1 { + fetchLimit = 0 + } + f.limit = fetchLimit + } + + return f } // Fetch attempts to fetch the file at the given path, from the distribution @@ -68,7 +69,9 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser return nil, fmt.Errorf("http.NewRequest error: %s", err) } - req.Header.Set("User-Agent", "go-ipfs") + if f.userAgent != "" { + req.Header.Set("User-Agent", f.userAgent) + } resp, err := http.DefaultClient.Do(req) if err != nil { diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index 9f107dbc2..ef1a65800 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -16,39 +16,12 @@ const ( envIpfsPath = "IPFS_PATH" defIpfsDir = ".ipfs" versionFile = "version" - - // Local IPFS API - apiFile = "api" ) func init() { homedir.DisableCache = true } -// ApiEndpoint reads the api file from the local ipfs install directory and -// returns the address:port read from the file. If the ipfs directory is not -// specified then the default location is used. -func ApiEndpoint(ipfsDir string) (string, error) { - ipfsDir, err := CheckIpfsDir(ipfsDir) - if err != nil { - return "", err - } - apiPath := path.Join(ipfsDir, apiFile) - - apiData, err := ioutil.ReadFile(apiPath) - if err != nil { - return "", err - } - - val := strings.TrimSpace(string(apiData)) - parts := strings.Split(val, "/") - if len(parts) != 5 { - return "", fmt.Errorf("incorrectly formatted api string: %q", val) - } - - return parts[2] + ":" + parts[4], nil -} - // IpfsDir returns the path of the ipfs directory. If dir specified, then // returns the expanded version dir. If dir is "", then return the directory // set by IPFS_PATH, or if IPFS_PATH is not set, then return the default diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index a1003bc7d..2ccd507e3 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -158,58 +158,3 @@ func testRepoVersion(t *testing.T) { t.Fatal(err) } } - -func TestApiEndpoint(t *testing.T) { - var err error - fakeHome, err = ioutil.TempDir("", "testhome") - if err != nil { - panic(err) - } - defer os.RemoveAll(fakeHome) - defer os.Unsetenv("HOME") - - os.Setenv("HOME", fakeHome) - fakeIpfs = path.Join(fakeHome, ".ipfs") - - err = os.Mkdir(fakeIpfs, os.ModePerm) - if err != nil { - panic(err) - } - - _, err = ApiEndpoint("") - if err == nil { - t.Fatal("expected error when missing api file") - } - - apiPath := path.Join(fakeIpfs, apiFile) - err = ioutil.WriteFile(apiPath, []byte("bad-data"), 0644) - if err != nil { - panic(err) - } - - _, err = ApiEndpoint("") - if err == nil { - t.Fatal("expected error when bad data") - } - - err = ioutil.WriteFile(apiPath, []byte("/ip4/127.0.0.1/tcp/5001"), 0644) - if err != nil { - panic(err) - } - - val, err := ApiEndpoint("") - if err != nil { - t.Fatal(err) - } - if val != "127.0.0.1:5001" { - t.Fatal("got unexpected value:", val) - } - - val2, err := ApiEndpoint(fakeIpfs) - if err != nil { - t.Fatal(err) - } - if val2 != val { - t.Fatal("expected", val, "got", val2) - } -} diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 51e476100..089eaa8d8 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -115,14 +115,9 @@ func TestFetchMigrations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewHttpFetcher() - fetcher.SetDistPath(CurrentIpfsDist) ts := createTestServer() defer ts.Close() - err := fetcher.SetGateway(ts.URL) - if err != nil { - panic(err) - } + fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) tmpDir, err := ioutil.TempDir("", "migratetest") if err != nil { @@ -166,16 +161,12 @@ func TestRunMigrations(t *testing.T) { t.Fatal(err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fetcher := NewHttpFetcher() - fetcher.SetDistPath(CurrentIpfsDist) ts := createTestServer() defer ts.Close() - if err = fetcher.SetGateway(ts.URL); err != nil { - panic(err) - } + fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() targetVer := 9 diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go index 7d9fda835..508e1e01f 100644 --- a/repo/fsrepo/migrations/versions.go +++ b/repo/fsrepo/migrations/versions.go @@ -5,10 +5,8 @@ import ( "context" "errors" "fmt" - "os/exec" "path" "sort" - "strconv" "strings" "github.com/coreos/go-semver/semver" @@ -75,19 +73,3 @@ func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bo return out, nil } - -// IpfsRepoVersion returns the repo version required by the ipfs daemon -func IpfsRepoVersion(ctx context.Context) (int, error) { - out, err := exec.CommandContext(ctx, "ipfs", "version", "--repo").CombinedOutput() - if err != nil { - return 0, fmt.Errorf("%s: %s", err, string(out)) - } - - verStr := strings.TrimSpace(string(out)) - ver, err := strconv.Atoi(verStr) - if err != nil { - return 0, fmt.Errorf("repo version is not an integer: %s", verStr) - } - - return ver, nil -} diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index 984d70abd..130a6de26 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -2,7 +2,6 @@ package migrations import ( "context" - "os/exec" "testing" "github.com/coreos/go-semver/semver" @@ -14,13 +13,9 @@ func TestDistVersions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewHttpFetcher() ts := createTestServer() defer ts.Close() - err := fetcher.SetGateway(ts.URL) - if err != nil { - panic(err) - } + fetcher := NewHttpFetcher("", ts.URL, "", 0) vers, err := DistVersions(ctx, fetcher, testDist, true) if err != nil { @@ -37,10 +32,9 @@ func TestLatestDistVersion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fetcher := NewHttpFetcher() - //ts := createTestServer() - //defer ts.Close() - //fetcher.SetGateway(ts.URL) + ts := createTestServer() + defer ts.Close() + fetcher := NewHttpFetcher("", ts.URL, "", 0) latest, err := LatestDistVersion(ctx, fetcher, testDist, false) if err != nil { @@ -55,22 +49,3 @@ func TestLatestDistVersion(t *testing.T) { } t.Log("Latest version of", testDist, "is", latest) } - -func TestIpfsRepoVersion(t *testing.T) { - _, err := exec.LookPath("ipfs") - if err != nil { - t.Skip("ipfs not available") - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ipfsRepoVer, err := IpfsRepoVersion(ctx) - if err != nil { - t.Fatal("Could not get required repo version:", err) - } - if ipfsRepoVer < 1 { - t.Fatal("Invalid repo version") - } - t.Log("IPFS repo version:", ipfsRepoVer) -} From 1430a0104c79eeddbfaa9e14cf2bb07ed73be9c2 Mon Sep 17 00:00:00 2001 From: gammazero Date: Tue, 16 Mar 2021 16:57:44 -0700 Subject: [PATCH 14/23] Update name of latest distribution path const --- repo/fsrepo/migrations/fetch_test.go | 2 +- repo/fsrepo/migrations/fetcher.go | 6 +++--- repo/fsrepo/migrations/httpfetcher.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 7441e248e..4b5a48bd9 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -55,7 +55,7 @@ func createFakeArchive(name string, archZip bool, w io.Writer) { func TestGetDistPath(t *testing.T) { os.Unsetenv(envIpfsDistPath) distPath := GetDistPathEnv("") - if distPath != IpnsIpfsDist { + if distPath != LatestIpfsDist { t.Error("did not set default dist path") } diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 23a2370ce..4bedafb63 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -9,8 +9,8 @@ import ( const ( // Current dirstibution to fetch migrations from CurrentIpfsDist = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" - // Distribution IPNS path. Default for fetchers. - IpnsIpfsDist = "/ipns/dist.ipfs.io" + // Latest distribution path. Default for fetchers. + LatestIpfsDist = "/ipns/dist.ipfs.io" // Distribution environ variable envIpfsDistPath = "IPFS_DIST_PATH" @@ -77,7 +77,7 @@ func GetDistPathEnv(distPath string) string { return dist } if distPath == "" { - return IpnsIpfsDist + return LatestIpfsDist } return distPath } diff --git a/repo/fsrepo/migrations/httpfetcher.go b/repo/fsrepo/migrations/httpfetcher.go index efc43bef6..876457f10 100644 --- a/repo/fsrepo/migrations/httpfetcher.go +++ b/repo/fsrepo/migrations/httpfetcher.go @@ -32,7 +32,7 @@ var _ Fetcher = (*HttpFetcher)(nil) // Specifying 0 for fetchLimit sets the default, -1 means no limit. func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *HttpFetcher { f := &HttpFetcher{ - distPath: IpnsIpfsDist, + distPath: LatestIpfsDist, gateway: defaultGatewayURL, limit: defaultFetchLimit, } From 1530cd2ceab3925d0203972869b0cbf296ecea72 Mon Sep 17 00:00:00 2001 From: Andrew Gillis Date: Thu, 25 Mar 2021 10:10:26 -0700 Subject: [PATCH 15/23] Rename migrations from ipfs-x-to-y to fs-repo-x-to-y (#8002) * rename migrations from ipfs-x-to-y to fs-repo-x-to-y * update current ipfs dist to one with migrations v1.0.0 --- repo/fsrepo/migrations/fetch.go | 4 ++-- repo/fsrepo/migrations/fetcher.go | 2 +- repo/fsrepo/migrations/migrations.go | 2 +- repo/fsrepo/migrations/migrations_test.go | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 753c9b10b..8cb6bedde 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -27,8 +27,8 @@ import ( // same name it has inside the archive. Otherwise, the binary file is written // to the file named by out. func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out string) (string, error) { - // The archive file name is the base of dist to support possible subdir in - // dist, for example: "ipfs-repo-migrations/ipfs-11-to-12" + // The archive file name is the base of dist. This is to support a possible subdir in + // dist, for example: "ipfs-repo-migrations/fs-repo-11-to-12" arcName := path.Base(dist) // If binary base name is not specified, then it is same as archive base name. if binName == "" { diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 4bedafb63..bf04cb3c0 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -8,7 +8,7 @@ import ( const ( // Current dirstibution to fetch migrations from - CurrentIpfsDist = "/ipfs/Qme8pJhBidEUXRdpcWLGR2fkG5kdwVnaMh3kabjfP8zz7Y" + CurrentIpfsDist = "/ipfs/QmZqzacg5Q8WpDL7SymogoaSZYw6RZT2kgPavymCmMwWse" // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.io" diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 1833dff30..0d51c6386 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -105,7 +105,7 @@ func ExeName(name string) string { } func migrationName(from, to int) string { - return fmt.Sprintf("ipfs-%d-to-%d", from, to) + return fmt.Sprintf("fs-repo-%d-to-%d", from, to) } // findMigrations returns a list of migrations, ordered from first to last diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 089eaa8d8..6ba02a2f7 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -125,7 +125,7 @@ func TestFetchMigrations(t *testing.T) { } defer os.RemoveAll(tmpDir) - needed := []string{"ipfs-1-to-2", "ipfs-2-to-3"} + needed := []string{"fs-repo-1-to-2", "fs-repo-2-to-3"} fetched, err := fetchMigrations(ctx, fetcher, needed, tmpDir) if err != nil { t.Fatal(err) @@ -177,7 +177,7 @@ func TestRunMigrations(t *testing.T) { err = RunMigration(ctx, fetcher, targetVer, fakeIpfs, true) if err != nil { - if !strings.HasPrefix(err.Error(), "migration ipfs-10-to-11 failed") { + if !strings.HasPrefix(err.Error(), "migration fs-repo-10-to-11 failed") { t.Fatal(err) } } From 09a481ea7a34d92bd2a4a7b4b94333b89d37ac85 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 25 Mar 2021 12:25:14 -0700 Subject: [PATCH 16/23] update current ipfs dist to migrations with vtag --- repo/fsrepo/migrations/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index bf04cb3c0..0ee66b533 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -8,7 +8,7 @@ import ( const ( // Current dirstibution to fetch migrations from - CurrentIpfsDist = "/ipfs/QmZqzacg5Q8WpDL7SymogoaSZYw6RZT2kgPavymCmMwWse" + CurrentIpfsDist = "/ipfs/QmWLyhqWDsWbcWE8vjmHkzGKLGgvHh84cLxM3ceLsojwrx" // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.io" From 891c81e631a559d831ba3687bfb48dd4d7af033c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 25 Mar 2021 15:53:15 -0400 Subject: [PATCH 17/23] test: repo migrations fetch - support windows wanting binaries to end in .exe --- repo/fsrepo/migrations/fetch_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 4b5a48bd9..7d90d7405 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -40,6 +40,7 @@ func createFakeArchive(name string, archZip bool, w io.Writer) { if fileName == "go-ipfs" { fileName = "ipfs" } + fileName = ExeName(fileName) var err error if archZip { @@ -179,7 +180,7 @@ func TestFetchBinary(t *testing.T) { t.Error("expected 'exists' error, got:", err) } - os.Remove(path.Join(tmpDir, "ipfs")) + os.Remove(path.Join(tmpDir, ExeName("ipfs"))) // Check error creating temp download directory err = os.Chmod(tmpDir, 0555) From 36de939e25fce2dd19118206b9f857de9bc76e98 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 25 Mar 2021 15:55:37 -0400 Subject: [PATCH 18/23] test: repo migrations fetch - skip part of a test on windows since it does not support read-only directories --- repo/fsrepo/migrations/fetch_test.go | 44 ++++++++++++++++------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index 7d90d7405..bf0b33c4f 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -10,6 +10,7 @@ import ( "net/http/httptest" "os" "path" + "runtime" "strings" "testing" ) @@ -183,25 +184,30 @@ func TestFetchBinary(t *testing.T) { os.Remove(path.Join(tmpDir, ExeName("ipfs"))) // Check error creating temp download directory - err = os.Chmod(tmpDir, 0555) - if err != nil { - panic(err) - } - err = os.Setenv("TMPDIR", tmpDir) - if err != nil { - panic(err) - } - _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) - if !os.IsPermission(err) { - t.Error("expected 'permission' error, got:", err) - } - err = os.Setenv("TMPDIR", "/tmp") - if err != nil { - panic(err) - } - err = os.Chmod(tmpDir, 0755) - if err != nil { - panic(err) + // + // Windows doesn't have read-only directories https://github.com/golang/go/issues/35042 this would need to be + // tested another way + if runtime.GOOS != "windows" { + err = os.Chmod(tmpDir, 0555) + if err != nil { + panic(err) + } + err = os.Setenv("TMPDIR", tmpDir) + if err != nil { + panic(err) + } + _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir) + if !os.IsPermission(err) { + t.Error("expected 'permission' error, got:", err) + } + err = os.Setenv("TMPDIR", "/tmp") + if err != nil { + panic(err) + } + err = os.Chmod(tmpDir, 0755) + if err != nil { + panic(err) + } } // Check error if failure to fetch due to bad dist From 5ee8710c5806bb26982b82a1567a28c75d2aed05 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 25 Mar 2021 15:47:45 -0400 Subject: [PATCH 19/23] feat: repo migrations - use filepath instead of path to support windows --- repo/fsrepo/migrations/fetch.go | 8 ++++---- repo/fsrepo/migrations/fetch_test.go | 3 ++- repo/fsrepo/migrations/ipfsdir.go | 8 ++++---- repo/fsrepo/migrations/ipfsdir_test.go | 6 +++--- repo/fsrepo/migrations/migrations_test.go | 6 +++--- repo/fsrepo/migrations/unpack_test.go | 13 +++++++------ 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index 8cb6bedde..0682aea53 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -9,7 +9,7 @@ import ( "io/ioutil" "os" "os/exec" - "path" + "path/filepath" "runtime" "strings" ) @@ -29,7 +29,7 @@ import ( func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out string) (string, error) { // The archive file name is the base of dist. This is to support a possible subdir in // dist, for example: "ipfs-repo-migrations/fs-repo-11-to-12" - arcName := path.Base(dist) + arcName := filepath.Base(dist) // If binary base name is not specified, then it is same as archive base name. if binName == "" { binName = arcName @@ -53,7 +53,7 @@ func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out s } } // out exists and is a directory, so compose final name - out = path.Join(out, binName) + out = filepath.Join(out, binName) // Check if the binary already exists in the directory _, err = os.Stat(out) if !os.IsNotExist(err) { @@ -83,7 +83,7 @@ func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out s arcDistPath, arcFullName := makeArchivePath(dist, arcName, ver, atype) // Create a file to write the archive data to - arcPath := path.Join(tmpDir, arcFullName) + arcPath := filepath.Join(tmpDir, arcFullName) arcFile, err := os.Create(arcPath) if err != nil { return "", err diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index bf0b33c4f..36cfb4662 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -10,6 +10,7 @@ import ( "net/http/httptest" "os" "path" + "path/filepath" "runtime" "strings" "testing" @@ -181,7 +182,7 @@ func TestFetchBinary(t *testing.T) { t.Error("expected 'exists' error, got:", err) } - os.Remove(path.Join(tmpDir, ExeName("ipfs"))) + os.Remove(filepath.Join(tmpDir, ExeName("ipfs"))) // Check error creating temp download directory // diff --git a/repo/fsrepo/migrations/ipfsdir.go b/repo/fsrepo/migrations/ipfsdir.go index ef1a65800..241e1ab8d 100644 --- a/repo/fsrepo/migrations/ipfsdir.go +++ b/repo/fsrepo/migrations/ipfsdir.go @@ -5,7 +5,7 @@ import ( "fmt" "io/ioutil" "os" - "path" + "path/filepath" "strconv" "strings" @@ -47,7 +47,7 @@ func IpfsDir(dir string) (string, error) { return "", errors.New("could not determine IPFS_PATH, home dir not set") } - return path.Join(home, defIpfsDir), nil + return filepath.Join(home, defIpfsDir), nil } // CheckIpfsDir gets the ipfs directory and checks that the directory exists. @@ -84,12 +84,12 @@ func WriteRepoVersion(ipfsDir string, version int) error { return err } - vFilePath := path.Join(ipfsDir, versionFile) + vFilePath := filepath.Join(ipfsDir, versionFile) return ioutil.WriteFile(vFilePath, []byte(fmt.Sprintf("%d\n", version)), 0644) } func repoVersion(ipfsDir string) (int, error) { - c, err := ioutil.ReadFile(path.Join(ipfsDir, versionFile)) + c, err := ioutil.ReadFile(filepath.Join(ipfsDir, versionFile)) if err != nil { return 0, err } diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index 2ccd507e3..2fecefa92 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -3,7 +3,7 @@ package migrations import ( "io/ioutil" "os" - "path" + "path/filepath" "testing" ) @@ -21,7 +21,7 @@ func TestRepoDir(t *testing.T) { defer os.RemoveAll(fakeHome) os.Setenv("HOME", fakeHome) - fakeIpfs = path.Join(fakeHome, ".ipfs") + fakeIpfs = filepath.Join(fakeHome, ".ipfs") t.Run("testIpfsDir", testIpfsDir) t.Run("testCheckIpfsDir", testCheckIpfsDir) @@ -144,7 +144,7 @@ func testRepoVersion(t *testing.T) { if err != nil { t.Fatal(err) } - vFilePath := path.Join(ipfsDir, versionFile) + vFilePath := filepath.Join(ipfsDir, versionFile) err = ioutil.WriteFile(vFilePath, []byte("bad-version-data\n"), 0644) if err != nil { panic(err) diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index 6ba02a2f7..812e5bd76 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -4,7 +4,7 @@ import ( "context" "io/ioutil" "os" - "path" + "path/filepath" "strings" "testing" ) @@ -148,7 +148,7 @@ func TestRunMigrations(t *testing.T) { defer os.RemoveAll(fakeHome) os.Setenv("HOME", fakeHome) - fakeIpfs := path.Join(fakeHome, ".ipfs") + fakeIpfs := filepath.Join(fakeHome, ".ipfs") err = os.Mkdir(fakeIpfs, os.ModePerm) if err != nil { @@ -184,7 +184,7 @@ func TestRunMigrations(t *testing.T) { } func createFakeBin(from, to int, tmpDir string) { - migPath := path.Join(tmpDir, ExeName(migrationName(from, to))) + migPath := filepath.Join(tmpDir, ExeName(migrationName(from, to))) emptyFile, err := os.Create(migPath) if err != nil { panic(err) diff --git a/repo/fsrepo/migrations/unpack_test.go b/repo/fsrepo/migrations/unpack_test.go index e772f00b1..f4266b024 100644 --- a/repo/fsrepo/migrations/unpack_test.go +++ b/repo/fsrepo/migrations/unpack_test.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strings" "testing" ) @@ -38,7 +39,7 @@ func TestUnpackTgz(t *testing.T) { } defer os.RemoveAll(tmpDir) - badTarGzip := path.Join(tmpDir, "bad.tar.gz") + badTarGzip := filepath.Join(tmpDir, "bad.tar.gz") err = ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644) if err != nil { panic(err) @@ -48,14 +49,14 @@ func TestUnpackTgz(t *testing.T) { t.Fatal("expected error opening gzip reader, got:", err) } - testTarGzip := path.Join(tmpDir, "test.tar.gz") + testTarGzip := filepath.Join(tmpDir, "test.tar.gz") testData := "some data" err = writeTarGzipFile(testTarGzip, "testroot", "testfile", testData) if err != nil { panic(err) } - out := path.Join(tmpDir, "out.txt") + out := filepath.Join(tmpDir, "out.txt") // Test looking for file that is not in archive err = unpackTgz(testTarGzip, "testroot", "abc", out) @@ -86,7 +87,7 @@ func TestUnpackZip(t *testing.T) { } defer os.RemoveAll(tmpDir) - badZip := path.Join(tmpDir, "bad.zip") + badZip := filepath.Join(tmpDir, "bad.zip") err = ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644) if err != nil { panic(err) @@ -96,14 +97,14 @@ func TestUnpackZip(t *testing.T) { t.Fatal("expected error opening zip reader, got:", err) } - testZip := path.Join(tmpDir, "test.zip") + testZip := filepath.Join(tmpDir, "test.zip") testData := "some data" err = writeZipFile(testZip, "testroot", "testfile", testData) if err != nil { panic(err) } - out := path.Join(tmpDir, "out.txt") + out := filepath.Join(tmpDir, "out.txt") // Test looking for file that is not in archive err = unpackZip(testZip, "testroot", "abc", out) From f00b49be5855121fd257bc6bb9572530a3e8eb2e Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 25 Mar 2021 23:42:59 -0700 Subject: [PATCH 20/23] Set latest dist to use one with darwin arm64 builds --- repo/fsrepo/migrations/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 0ee66b533..cb2900471 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -8,7 +8,7 @@ import ( const ( // Current dirstibution to fetch migrations from - CurrentIpfsDist = "/ipfs/QmWLyhqWDsWbcWE8vjmHkzGKLGgvHh84cLxM3ceLsojwrx" + CurrentIpfsDist = "/ipfs/Qmaubnx6vDUEA2arLzPWxqY2brx2c1CUKsrgQHSwBXDZ5E" // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.io" From fe97eee1c90bc8ca29bc84753b7c723615688f2d Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 26 Mar 2021 00:48:13 -0700 Subject: [PATCH 21/23] Fix typo in log message --- repo/fsrepo/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 0d51c6386..f8d343d65 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -54,7 +54,7 @@ func RunMigration(ctx context.Context, fetcher Fetcher, targetVer int, ipfsDir s } } - log.Print("Need", len(missing), "migrations, downloading.") + log.Println("Need", len(missing), "migrations, downloading.") tmpDir, err := ioutil.TempDir("", "migrations") if err != nil { From 2fc1594f51a606d2f1e1b4b735fa5c8098153c4f Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 26 Mar 2021 01:47:24 -0700 Subject: [PATCH 22/23] Use same semver lib that is use elsewhere in go-ipfs --- go.mod | 4 ++-- go.sum | 6 ++++-- repo/fsrepo/migrations/versions.go | 6 +++--- repo/fsrepo/migrations/versions_test.go | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 190ed2675..5b68ecaa2 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc contrib.go.opencensus.io/exporter/prometheus v0.2.0 github.com/blang/semver v3.5.1+incompatible + github.com/blang/semver/v4 v4.0.0 github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d github.com/cheggaaa/pb v1.0.29 - github.com/coreos/go-semver v0.3.0 - github.com/coreos/go-systemd/v22 v22.1.0 + github.com/coreos/go-systemd/v22 v22.3.0 github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/fsnotify/fsnotify v1.4.9 diff --git a/go.sum b/go.sum index d1141d651..51746c3ed 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d h1:QgeLLoPD3kRVmeu/1al9iIpIANMi9O1zXFm8BnYGCJg= github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d/go.mod h1:Jbj8eKecMNwf0KFI75skSUZqMB4UCRcndUScVBTWyUI= @@ -103,8 +105,8 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.0 h1:C8u/Ljj8G8O6rqWJh2J8cDyeEFBMWvXlvJ/ccMyzcqw= +github.com/coreos/go-systemd/v22 v22.3.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go index 508e1e01f..69b2e290b 100644 --- a/repo/fsrepo/migrations/versions.go +++ b/repo/fsrepo/migrations/versions.go @@ -9,7 +9,7 @@ import ( "sort" "strings" - "github.com/coreos/go-semver/semver" + "github.com/blang/semver/v4" ) const distVersions = "versions" @@ -46,11 +46,11 @@ func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bo defer rc.Close() prefix := "v" - var vers []*semver.Version + var vers []semver.Version scan := bufio.NewScanner(rc) for scan.Scan() { - ver, err := semver.NewVersion(strings.TrimLeft(scan.Text(), prefix)) + ver, err := semver.Make(strings.TrimLeft(scan.Text(), prefix)) if err != nil { continue } diff --git a/repo/fsrepo/migrations/versions_test.go b/repo/fsrepo/migrations/versions_test.go index 130a6de26..18de72b77 100644 --- a/repo/fsrepo/migrations/versions_test.go +++ b/repo/fsrepo/migrations/versions_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - "github.com/coreos/go-semver/semver" + "github.com/blang/semver/v4" ) const testDist = "go-ipfs" @@ -43,7 +43,7 @@ func TestLatestDistVersion(t *testing.T) { if len(latest) < 6 { t.Fatal("latest version string too short", latest) } - _, err = semver.NewVersion(latest[1:]) + _, err = semver.New(latest[1:]) if err != nil { t.Fatal("latest version has invalid format:", latest) } From b75d8239504280769228f1a54162e09abc32f7f8 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 31 Mar 2021 12:25:32 -0700 Subject: [PATCH 23/23] Update latest distribution CID --- repo/fsrepo/migrations/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index cb2900471..f6d5f6686 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -8,7 +8,7 @@ import ( const ( // Current dirstibution to fetch migrations from - CurrentIpfsDist = "/ipfs/Qmaubnx6vDUEA2arLzPWxqY2brx2c1CUKsrgQHSwBXDZ5E" + CurrentIpfsDist = "/ipfs/QmVxxcTSuryJYdQJGcS8SyhzN7NBNLTqVPAxpu6gp2ZcrR" // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.io"