Fix error reading zip when archive not found

- Udate path to IPFS dist
- Improve test coverage

repo/fsrepo/migrations/fetch.go

@@ -16,6 +16,8 @@ import (
 )
 
 const (
+	envIpfsDistPath = "IPFS_DIST_PATH"
+
 	// Distribution
 	gatewayURL = "https://ipfs.io"
 	ipfsDist   = "/ipns/dist.ipfs.io"
@@ -45,7 +47,7 @@ func SetIpfsDistPath(distPath string) {
 		return
 	}
 
-	if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {
+	if dist := os.Getenv(envIpfsDistPath); dist != "" {
 		ipfsDistPath = dist
 	} else {
 		ipfsDistPath = ipfsDist

repo/fsrepo/migrations/fetch_test.go

@@ -10,6 +10,35 @@ import (
 	"testing"
 )
 
+func TestSetIpfsDistPath(t *testing.T) {
+	os.Unsetenv(envIpfsDistPath)
+	SetIpfsDistPath("")
+	if ipfsDistPath != ipfsDist {
+		t.Error("did not set default dist path")
+	}
+
+	testDist := "/unit/test/dist"
+	err := os.Setenv(envIpfsDistPath, testDist)
+	if err != nil {
+		panic(err)
+	}
+	defer func() {
+		os.Unsetenv(envIpfsDistPath)
+		SetIpfsDistPath("")
+	}()
+
+	SetIpfsDistPath("")
+	if ipfsDistPath != testDist {
+		t.Error("did not set dist path from environ")
+	}
+
+	testDist = "/unit/test/dist2"
+	SetIpfsDistPath(testDist)
+	if ipfsDistPath != testDist {
+		t.Error("did not set dist path")
+	}
+}
+
 func TestHttpFetch(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -39,11 +68,24 @@ func TestHttpFetch(t *testing.T) {
 	}
 
 	// Check bad URL
+	_, err = httpFetch(ctx, "")
+	if err == nil {
+		t.Fatal("expected error")
+	}
+
+	// Check unreachable URL
+	_, err = httpFetch(ctx, "http://127.0.0.123:65510")
+	if err == nil || !strings.HasSuffix(err.Error(), "connection refused") {
+		t.Fatal("expected 'connection refused' error")
+	}
+
+	// Check not found
 	url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file")
 	_, err = httpFetch(ctx, url)
 	if err == nil || !strings.Contains(err.Error(), "404") {
 		t.Fatal("expected error 404")
 	}
+
 }
 
 func TestIpfsFetch(t *testing.T) {
@@ -103,7 +145,7 @@ func TestFetchBinary(t *testing.T) {
 	}
 	t.Log("latest version of", distFSRM, "is", vers[len(vers)-1])
 
-	bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, distFSRM, tmpDir)
+	bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, "", tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -115,7 +157,7 @@ func TestFetchBinary(t *testing.T) {
 
 	t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
 
-	bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "go-ipfs", "ipfs", tmpDir)
+	bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -126,4 +168,44 @@ func TestFetchBinary(t *testing.T) {
 	}
 
 	t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
+
+	// Check error is destination already exists and is not directory
+	_, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", bin)
+	if !os.IsExist(err) {
+		t.Fatal("expected 'exists' error")
+	}
+
+	// Check error creating temp download directory
+	err = os.Chmod(tmpDir, 0555)
+	if err != nil {
+		panic(err)
+	}
+	err = os.Setenv("TMPDIR", tmpDir)
+	if err != nil {
+		panic(err)
+	}
+	_, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir)
+	if !os.IsPermission(err) {
+		t.Error("expected 'permission'error")
+	}
+	err = os.Setenv("TMPDIR", "/tmp")
+	if err != nil {
+		panic(err)
+	}
+	err = os.Chmod(tmpDir, 0755)
+	if err != nil {
+		panic(err)
+	}
+
+	// Check error if failure to fetch due to bad dist
+	_, err = FetchBinary(ctx, "no-such-dist", "v0.3.5", "", "ipfs", tmpDir)
+	if err == nil || !strings.Contains(err.Error(), "Not Found") {
+		t.Error("expected 'Not Found' error")
+	}
+
+	// Check error if failure to unpack archive
+	_, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "not-such-bin", tmpDir)
+	if err == nil || err.Error() != "no binary found in archive" {
+		t.Error("expected 'no binary found in archive' error")
+	}
 }

repo/fsrepo/migrations/ipfsdir_test.go

@@ -23,12 +23,12 @@ func TestRepoDir(t *testing.T) {
 	os.Setenv("HOME", fakeHome)
 	fakeIpfs = path.Join(fakeHome, ".ipfs")
 
-	t.Run("testFindIpfsDir", testFindIpfsDir)
+	t.Run("testIpfsDir", testIpfsDir)
 	t.Run("testCheckIpfsDir", testCheckIpfsDir)
 	t.Run("testRepoVersion", testRepoVersion)
 }
 
-func testFindIpfsDir(t *testing.T) {
+func testIpfsDir(t *testing.T) {
 	_, err := CheckIpfsDir("")
 	if err == nil {
 		t.Fatal("expected error when no .ipfs directory to find")
@@ -47,7 +47,7 @@ func testFindIpfsDir(t *testing.T) {
 		t.Fatal("wrong ipfs directory:", dir)
 	}
 
-	os.Setenv("IPFS_PATH", "~/.ipfs")
+	os.Setenv(envIpfsPath, "~/.ipfs")
 	dir, err = IpfsDir("")
 	if err != nil {
 		t.Fatal(err)
@@ -55,10 +55,46 @@ func testFindIpfsDir(t *testing.T) {
 	if dir != fakeIpfs {
 		t.Fatal("wrong ipfs directory:", dir)
 	}
+
+	_, err = IpfsDir("~somesuer/foo")
+	if err == nil {
+		t.Fatal("expected error with user-specific home dir")
+	}
+
+	err = os.Setenv(envIpfsPath, "~somesuer/foo")
+	if err != nil {
+		panic(err)
+	}
+	_, err = IpfsDir("~somesuer/foo")
+	if err == nil {
+		t.Fatal("expected error with user-specific home dir")
+	}
+	err = os.Unsetenv(envIpfsPath)
+	if err != nil {
+		panic(err)
+	}
+
+	dir, err = IpfsDir("~/.ipfs")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dir != fakeIpfs {
+		t.Fatal("wrong ipfs directory:", dir)
+	}
+
+	_, err = IpfsDir("")
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func testCheckIpfsDir(t *testing.T) {
-	_, err := CheckIpfsDir("~/no_such_dir")
+	_, err := CheckIpfsDir("~somesuer/foo")
+	if err == nil {
+		t.Fatal("expected error with user-specific home dir")
+	}
+
+	_, err = CheckIpfsDir("~/no_such_dir")
 	if err == nil {
 		t.Fatal("expected error from nonexistent directory")
 	}
@@ -73,7 +109,13 @@ func testCheckIpfsDir(t *testing.T) {
 }
 
 func testRepoVersion(t *testing.T) {
-	_, err := RepoVersion(fakeIpfs)
+	badDir := "~somesuer/foo"
+	_, err := RepoVersion(badDir)
+	if err == nil {
+		t.Fatal("expected error with user-specific home dir")
+	}
+
+	_, err = RepoVersion(fakeIpfs)
 	if !os.IsNotExist(err) {
 		t.Fatal("expected not-exist error")
 	}
@@ -92,6 +134,29 @@ func testRepoVersion(t *testing.T) {
 	if ver != testVer {
 		t.Fatalf("expected version %d, got %d", testVer, ver)
 	}
+
+	err = WriteRepoVersion(badDir, testVer)
+	if err == nil {
+		t.Fatal("expected error with user-specific home dir")
+	}
+
+	ipfsDir, err := IpfsDir(fakeIpfs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	vFilePath := path.Join(ipfsDir, versionFile)
+	err = ioutil.WriteFile(vFilePath, []byte("bad-version-data\n"), 0644)
+	if err != nil {
+		panic(err)
+	}
+	_, err = RepoVersion(fakeIpfs)
+	if err == nil || err.Error() != "invalid data in repo version file" {
+		t.Fatal("expected 'invalid data' error")
+	}
+	err = WriteRepoVersion(fakeIpfs, testVer)
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestApiEndpoint(t *testing.T) {
@@ -147,4 +212,11 @@ func TestApiEndpoint(t *testing.T) {
 	if val2 != val {
 		t.Fatal("expected", val, "got", val2)
 	}
+
+	_, _, err = ApiShell(fakeIpfs)
+	if err != nil {
+		if err.Error() != "ipfs api shell not up" {
+			t.Fatal("expected 'ipfs api shell not up' error")
+		}
+	}
 }

repo/fsrepo/migrations/migrations_test.go

@@ -116,7 +116,7 @@ func TestFetchMigrations(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	SetIpfsDistPath("/ipfs/QmdFVsmD668ijuBFJwjyXdP5Sq44a5bPNAq3nnQF77kpyJ")
+	SetIpfsDistPath("/ipfs/QmXt92hFRuvQgFhgHoaMxC4wLFcvKsCywQPTNmPYCGfEV4")
 	_, err := LatestDistVersion(ctx, "ipfs-1-to-2")
 	if err != nil {
 		if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) {

repo/fsrepo/migrations/unpack.go

@@ -83,9 +83,14 @@ func unpackZip(arcPath, root, name, out string) error {
 			}
 
 			bin = rc
+			break
 		}
 	}
 
+	if bin == nil {
+		return errors.New("no binary found in archive")
+	}
+
 	return writeToPath(bin, out)
 }
 

repo/fsrepo/migrations/unpack_test.go

@@ -0,0 +1,212 @@
+package migrations
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bufio"
+	"compress/gzip"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"testing"
+)
+
+func TestUnpackArchive(t *testing.T) {
+	// Check unrecognized archive type
+	err := unpackArchive("", "no-arch-type", "", "", "")
+	if err == nil || err.Error() != "unrecognized archive type: no-arch-type" {
+		t.Fatal("expected 'unrecognized archive type' error")
+	}
+
+	// Test cannot open errors
+	err = unpackArchive("no-archive", "tar.gz", "", "", "")
+	if err == nil || !strings.HasPrefix(err.Error(), "cannot open archive file") {
+		t.Fatal("expected 'cannot open' error, got:", err)
+	}
+	err = unpackArchive("no-archive", "zip", "", "", "")
+	if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") {
+		t.Fatal("expected 'cannot open' error, got:", err)
+	}
+}
+
+func TestUnpackTgz(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "testunpacktgz")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	badTarGzip := path.Join(tmpDir, "bad.tar.gz")
+	err = ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644)
+	if err != nil {
+		panic(err)
+	}
+	err = unpackTgz(badTarGzip, "", "abc", "abc")
+	if err == nil || !strings.HasPrefix(err.Error(), "error opening gzip reader") {
+		t.Fatal("expected error opening gzip reader, got:", err)
+	}
+
+	testTarGzip := path.Join(tmpDir, "test.tar.gz")
+	testData := "some data"
+	err = writeTarGzip(testTarGzip, "testroot", "testfile", testData)
+	if err != nil {
+		panic(err)
+	}
+
+	out := path.Join(tmpDir, "out.txt")
+
+	// Test looking for file that is not in archive
+	err = unpackTgz(testTarGzip, "testroot", "abc", out)
+	if err == nil || err.Error() != "no binary found in archive" {
+		t.Fatal("expected 'no binary found in archive' error, got:", err)
+	}
+
+	// Test that unpack works.
+	err = unpackTgz(testTarGzip, "testroot", "testfile", out)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fi, err := os.Stat(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fi.Size() != int64(len(testData)) {
+		t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData))
+	}
+
+}
+
+func TestUnpackZip(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "testunpackzip")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	badZip := path.Join(tmpDir, "bad.zip")
+	err = ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644)
+	if err != nil {
+		panic(err)
+	}
+	err = unpackZip(badZip, "", "abc", "abc")
+	if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") {
+		t.Fatal("expected error opening zip reader, got:", err)
+	}
+
+	testZip := path.Join(tmpDir, "test.zip")
+	testData := "some data"
+	err = writeZip(testZip, "testroot", "testfile", testData)
+	if err != nil {
+		panic(err)
+	}
+
+	out := path.Join(tmpDir, "out.txt")
+
+	// Test looking for file that is not in archive
+	err = unpackZip(testZip, "testroot", "abc", out)
+	if err == nil || err.Error() != "no binary found in archive" {
+		t.Fatal("expected 'no binary found in archive' error, got:", err)
+	}
+
+	// Test that unpack works.
+	err = unpackZip(testZip, "testroot", "testfile", out)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fi, err := os.Stat(out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fi.Size() != int64(len(testData)) {
+		t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData))
+	}
+}
+
+func writeTarGzip(archName, root, fileName, data string) error {
+	archFile, err := os.Create(archName)
+	if err != nil {
+		return err
+	}
+	defer archFile.Close()
+	wr := bufio.NewWriter(archFile)
+
+	// gzip writer writes to buffer
+	gzw := gzip.NewWriter(wr)
+	defer gzw.Close()
+	// tar writer writes to gzip
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	if fileName != "" {
+		hdr := &tar.Header{
+			Name: path.Join(root, fileName),
+			Mode: 0600,
+			Size: int64(len(data)),
+		}
+		// Write header
+		if err = tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		// Write file body
+		if _, err := tw.Write([]byte(data)); err != nil {
+			return err
+		}
+	}
+
+	if err = tw.Close(); err != nil {
+		return err
+	}
+	// Close gzip writer; finish writing gzip data to buffer
+	if err = gzw.Close(); err != nil {
+		return err
+	}
+	// Flush buffered data to file
+	if err = wr.Flush(); err != nil {
+		return err
+	}
+	// Close tar file
+	if err = archFile.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func writeZip(archName, root, fileName, data string) error {
+	archFile, err := os.Create(archName)
+	if err != nil {
+		return err
+	}
+	defer archFile.Close()
+	wr := bufio.NewWriter(archFile)
+
+	zw := zip.NewWriter(wr)
+	defer zw.Close()
+
+	// Write file name
+	f, err := zw.Create(path.Join(root, fileName))
+	if err != nil {
+		return err
+	}
+	// Write file data
+	_, err = f.Write([]byte(data))
+	if err != nil {
+		return err
+	}
+
+	// Close zip writer
+	if err = zw.Close(); err != nil {
+		return err
+	}
+	// Flush buffered data to file
+	if err = wr.Flush(); err != nil {
+		return err
+	}
+	// Close zip file
+	if err = archFile.Close(); err != nil {
+		return err
+	}
+	return nil
+}