coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-02-21 10:27:46 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Expose additional migration APIs (#8153)

Browse Source 
* Expose additional migration APIs

Expose migration APIs for reading migration config and creating migration fetchers.  This allows implementation of commands and external applications that want to retrieve migrations according to the Migration portion of the IPFS config.

This change also moves some functionality that is specific to fetching migrations via IPFS into the `ipfsfetcher` package.
This commit is contained in:
Andrew Gillis 2021-07-30 11:27:57 -07:00 committed by GitHub
parent cab67f6b66
commit 3b6f57788b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 526 additions and 511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
cmd/ipfs/migration.go → cmd/ipfs/add_migrations.go
View File

@ -2,17 +2,13 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"
config "github.com/ipfs/go-ipfs-config"
"github.com/ipfs/go-ipfs-files"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/core/coreapi"
@ -24,140 +20,7 @@ import (
"github.com/libp2p/go-libp2p-core/peer"
)
// readMigrationConfig reads the migration config out of the config, avoiding
// reading anything other than the migration section. That way, we're free to
// make arbitrary changes to all _other_ sections in migrations.
func readMigrationConfig(repoRoot string) (*config.Migration, error) {
var cfg struct {
Migration config.Migration
}
cfgPath, err := config.Filename(repoRoot)
if err != nil {
return nil, err
}
cfgFile, err := os.Open(cfgPath)
if err != nil {
return nil, err
}
defer cfgFile.Close()
err = json.NewDecoder(cfgFile).Decode(&cfg)
if err != nil {
return nil, err
}
switch cfg.Migration.Keep {
case "":
cfg.Migration.Keep = config.DefaultMigrationKeep
case "discard", "cache", "keep":
default:
return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'")
}
if len(cfg.Migration.DownloadSources) == 0 {
cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources
}
return &cfg.Migration, nil
}
func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) {
if repoRoot == nil {
return
}
cfgPath, err := config.Filename(*repoRoot)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
cfgFile, err := os.Open(cfgPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
defer cfgFile.Close()
// Attempt to read bootstrap addresses
var bootstrapCfg struct {
Bootstrap []string
}
err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config")
} else {
bootstrap = bootstrapCfg.Bootstrap
}
if _, err = cfgFile.Seek(0, 0); err != nil {
fmt.Fprintln(os.Stderr, err)
}
// Attempt to read peers
var peeringCfg struct {
Peering config.Peering
}
err = json.NewDecoder(cfgFile).Decode(&peeringCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read peering from config")
} else {
peers = peeringCfg.Peering.Peers
}
return
}
// getMigrationFetcher creates one or more fetchers according to
// config.Migration.DownloadSources. If an IpfsFetcher is required, then
// bootstrap and peer information in read from the config file in repoRoot,
// unless repoRoot is nil.
func getMigrationFetcher(cfg *config.Migration, repoRoot *string) (migrations.Fetcher, error) {
const httpUserAgent = "go-ipfs"
// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)
var fetchers []migrations.Fetcher
for _, src := range cfg.DownloadSources {
src := strings.TrimSpace(src)
switch src {
case "IPFS", "ipfs":
bootstrap, peers := readIpfsConfig(repoRoot)
fetchers = append(fetchers, ipfsfetcher.NewIpfsFetcher(fetchDistPath, 0, bootstrap, peers))
case "HTTPS", "https", "HTTP", "http":
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, "", httpUserAgent, 0))
default:
u, err := url.Parse(src)
if err != nil {
return nil, fmt.Errorf("bad gateway address: %s", err)
}
switch u.Scheme {
case "":
u.Scheme = "https"
case "https", "http":
default:
return nil, errors.New("bad gateway address: url scheme must be http or https")
}
fetchers = append(fetchers, migrations.NewHttpFetcher(fetchDistPath, u.String(), httpUserAgent, 0))
case "":
// Ignore empty string
}
}
if len(fetchers) == 0 {
return nil, errors.New("no sources specified")
}
if len(fetchers) == 1 {
return fetchers[0], nil
}
// Wrap fetchers in a MultiFetcher to try them in order
return migrations.NewMultiFetcher(fetchers...), nil
}
// addMigrations adds any migration downloaded by the fetcher to the IPFS node
func addMigrations(ctx context.Context, node *core.IpfsNode, fetcher migrations.Fetcher, pin bool) error {
var fetchers []migrations.Fetcher
if mf, ok := fetcher.(*migrations.MultiFetcher); ok {

19
cmd/ipfs/daemon.go
View File

@ -30,6 +30,7 @@ import (
nodeMount "github.com/ipfs/go-ipfs/fuse/node"
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher"
sockets "github.com/libp2p/go-socket-activation"
cmds "github.com/ipfs/go-ipfs-cmds"
@ -294,12 +295,26 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
return fmt.Errorf("fs-repo requires migration")
}
migrationCfg, err := readMigrationConfig(cctx.ConfigRoot)
// Read Migration section of IPFS config
migrationCfg, err := migrations.ReadMigrationConfig(cctx.ConfigRoot)
if err != nil {
return err
}
fetcher, err = getMigrationFetcher(migrationCfg, &cctx.ConfigRoot)
// Define function to create IPFS fetcher. Do not supply an
// already-constructed IPFS fetcher, because this may be expensive and
// not needed according to migration config. Instead, supply a function
// to construct the particular IPFS fetcher implementation used here,
// which is called only if an IPFS fetcher is needed.
newIpfsFetcher := func(distPath string) migrations.Fetcher {
return ipfsfetcher.NewIpfsFetcher(distPath, 0, &cctx.ConfigRoot)
}
// Fetch migrations from current distribution, or location from environ
fetchDistPath := migrations.GetDistPathEnv(migrations.CurrentIpfsDist)
// Create fetchers according to migrationCfg.DownloadSources
fetcher, err = migrations.GetMigrationFetcher(migrationCfg.DownloadSources, fetchDistPath, newIpfsFetcher)
if err != nil {
return err
}

312
cmd/ipfs/migration_test.go
View File

@ -1,312 +0,0 @@
package main
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
config "github.com/ipfs/go-ipfs-config"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
"github.com/ipfs/go-ipfs/repo/fsrepo/migrations/ipfsfetcher"
)
var testConfig = `
{
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"],
"Keep": "cache"
},
"Peering": {
"Peers": [
{
"ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5",
"Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"]
}
]
}
}
`
func TestReadMigrationConfigDefaults(t *testing.T) {
tmpDir := makeConfig("{}")
defer os.RemoveAll(tmpDir)
cfg, err := readMigrationConfig(tmpDir)
if err != nil {
t.Fatal(err)
}
if cfg.Keep != config.DefaultMigrationKeep {
t.Error("expected default value for Keep")
}
if len(cfg.DownloadSources) != len(config.DefaultMigrationDownloadSources) {
t.Fatal("expected default number of download sources")
}
for i, src := range config.DefaultMigrationDownloadSources {
if cfg.DownloadSources[i] != src {
t.Errorf("wrong DownloadSource: %s", cfg.DownloadSources[i])
}
}
}
func TestReadMigrationConfigErrors(t *testing.T) {
tmpDir := makeConfig(`{"Migration": {"Keep": "badvalue"}}`)
defer os.RemoveAll(tmpDir)
_, err := readMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
if !strings.HasPrefix(err.Error(), "unknown") {
t.Fatal("did not get expected error:", err)
}
os.RemoveAll(tmpDir)
_, err = readMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
bootstrap, peers := readIpfsConfig(&tmpDir)
if bootstrap != nil {
t.Error("expected nil bootstrap")
}
if peers != nil {
t.Error("expected nil peers")
}
tmpDir = makeConfig(`}{`)
defer os.RemoveAll(tmpDir)
_, err = readMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
}
func TestReadMigrationConfig(t *testing.T) {
tmpDir := makeConfig(testConfig)
defer os.RemoveAll(tmpDir)
cfg, err := readMigrationConfig(tmpDir)
if err != nil {
t.Fatal(err)
}
if len(cfg.DownloadSources) != 4 {
t.Fatal("wrong number of DownloadSources")
}
expect := []string{"IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"}
for i := range expect {
if cfg.DownloadSources[i] != expect[i] {
t.Errorf("wrong DownloadSource at %d", i)
}
}
if cfg.Keep != "cache" {
t.Error("wrong value for Keep")
}
}
func TestReadIpfsConfig(t *testing.T) {
tmpDir := makeConfig(testConfig)
defer os.RemoveAll(tmpDir)
bootstrap, peers := readIpfsConfig(nil)
if bootstrap != nil || peers != nil {
t.Fatal("expected nil ipfs config items")
}
bootstrap, peers = readIpfsConfig(&tmpDir)
if len(bootstrap) != 2 {
t.Fatal("wrong number of bootstrap addresses")
}
if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" {
t.Fatal("wrong bootstrap address")
}
if len(peers) != 1 {
t.Fatal("wrong number of peers")
}
peer := peers[0]
if peer.ID.String() != "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5" {
t.Errorf("wrong ID for first peer")
}
if len(peer.Addrs) != 2 {
t.Error("wrong number of addrs for first peer")
}
}
func TestReadPartialIpfsConfig(t *testing.T) {
const (
configBadBootstrap = `
{
"Bootstrap": "unreadable",
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1"],
"Keep": "cache"
},
"Peering": {
"Peers": [
{
"ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5",
"Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"]
}
]
}
}
`
configBadPeers = `
{
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1"],
"Keep": "cache"
},
"Peering": "Unreadable-data"
}
`
)
tmpDir := makeConfig(configBadBootstrap)
defer os.RemoveAll(tmpDir)
bootstrap, peers := readIpfsConfig(&tmpDir)
if bootstrap != nil {
t.Fatal("expected nil bootstrap")
}
if len(peers) != 1 {
t.Fatal("wrong number of peers")
}
if len(peers[0].Addrs) != 2 {
t.Error("wrong number of addrs for first peer")
}
os.RemoveAll(tmpDir)
tmpDir = makeConfig(configBadPeers)
defer os.RemoveAll(tmpDir)
bootstrap, peers = readIpfsConfig(&tmpDir)
if peers != nil {
t.Fatal("expected nil peers")
}
if len(bootstrap) != 2 {
t.Fatal("wrong number of bootstrap addresses")
}
if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" {
t.Fatal("wrong bootstrap address")
}
}
func makeConfig(configData string) string {
tmpDir, err := ioutil.TempDir("", "migration_test")
if err != nil {
panic(err)
}
cfgFile, err := os.Create(filepath.Join(tmpDir, "config"))
if err != nil {
panic(err)
}
if _, err = cfgFile.Write([]byte(configData)); err != nil {
panic(err)
}
if err = cfgFile.Close(); err != nil {
panic(err)
}
return tmpDir
}
func TestGetMigrationFetcher(t *testing.T) {
var f migrations.Fetcher
var err error
cfg := &config.Migration{}
cfg.DownloadSources = []string{"ftp://bad.gateway.io"}
_, err = getMigrationFetcher(cfg, nil)
if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") {
t.Fatal("Expected bad gateway address error, got:", err)
}
cfg.DownloadSources = []string{"::bad.gateway.io"}
_, err = getMigrationFetcher(cfg, nil)
if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") {
t.Fatal("Expected bad gateway address error, got:", err)
}
cfg.DownloadSources = []string{"http://localhost"}
f, err = getMigrationFetcher(cfg, nil)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*migrations.HttpFetcher); !ok {
t.Fatal("expected HttpFetcher")
}
cfg.DownloadSources = []string{"ipfs"}
f, err = getMigrationFetcher(cfg, nil)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*ipfsfetcher.IpfsFetcher); !ok {
t.Fatal("expected IpfsFetcher")
}
cfg.DownloadSources = []string{"http"}
f, err = getMigrationFetcher(cfg, nil)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*migrations.HttpFetcher); !ok {
t.Fatal("expected HttpFetcher")
}
cfg.DownloadSources = []string{"IPFS", "HTTPS"}
f, err = getMigrationFetcher(cfg, nil)
if err != nil {
t.Fatal(err)
}
mf, ok := f.(*migrations.MultiFetcher)
if !ok {
t.Fatal("expected MultiFetcher")
}
if mf.Len() != 2 {
t.Fatal("expected 2 fetchers in MultiFetcher")
}
cfg.DownloadSources = []string{"ipfs", "https", "some.domain.io"}
f, err = getMigrationFetcher(cfg, nil)
if err != nil {
t.Fatal(err)
}
mf, ok = f.(*migrations.MultiFetcher)
if !ok {
t.Fatal("expected MultiFetcher")
}
if mf.Len() != 3 {
t.Fatal("expected 3 fetchers in MultiFetcher")
}
cfg.DownloadSources = nil
_, err = getMigrationFetcher(cfg, nil)
if err == nil {
t.Fatal("expected error when no sources specified")
}
cfg.DownloadSources = []string{"", ""}
_, err = getMigrationFetcher(cfg, nil)
if err == nil {
t.Fatal("expected error when empty string fetchers specified")
}
}

6
repo/fsrepo/migrations/fetch_test.go
View File

@ -127,11 +127,7 @@ func TestHttpFetch(t *testing.T) {
}
func TestFetchBinary(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "fetchtest")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

8
repo/fsrepo/migrations/ipfsdir_test.go
View File

@ -13,13 +13,7 @@ var (
)
func TestRepoDir(t *testing.T) {
var err error
fakeHome, err = ioutil.TempDir("", "testhome")
if err != nil {
panic(err)
}
defer os.RemoveAll(fakeHome)
fakeHome = t.TempDir()
os.Setenv("HOME", fakeHome)
fakeIpfs = filepath.Join(fakeHome, ".ipfs")

73
repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go
View File

@ -2,6 +2,7 @@ package ipfsfetcher
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@ -32,10 +33,9 @@ const (
)
type IpfsFetcher struct {
distPath string
limit int64
bootstrap []string
peers []peer.AddrInfo
distPath string
limit int64
repoRoot *string
openOnce sync.Once
openErr error
@ -56,12 +56,15 @@ type IpfsFetcher struct {
//
// Specifying "" for distPath sets the default IPNS path.
// Specifying 0 for fetchLimit sets the default, -1 means no limit.
func NewIpfsFetcher(distPath string, fetchLimit int64, bootstrap []string, peers []peer.AddrInfo) *IpfsFetcher {
//
// Bootstrap and peer information in read from the IPFS config file in
// repoRoot, unless repoRoot is nil. If repoRoot is empty (""), then read the
// config from the default IPFS directory.
func NewIpfsFetcher(distPath string, fetchLimit int64, repoRoot *string) *IpfsFetcher {
f := &IpfsFetcher{
limit: defaultFetchLimit,
distPath: migrations.LatestIpfsDist,
bootstrap: bootstrap,
peers: peers,
limit: defaultFetchLimit,
distPath: migrations.LatestIpfsDist,
repoRoot: repoRoot,
}
if distPath != "" {
@ -88,7 +91,8 @@ func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser
// Initialize and start IPFS node on first call to Fetch, since the fetcher
// may be created by not used.
f.openOnce.Do(func() {
f.ipfsTmpDir, f.openErr = initTempNode(ctx, f.bootstrap, f.peers)
bootstrap, peers := readIpfsConfig(f.repoRoot)
f.ipfsTmpDir, f.openErr = initTempNode(ctx, bootstrap, peers)
if f.openErr != nil {
return
}
@ -277,3 +281,52 @@ func parsePath(fetchPath string) (ipath.Path, error) {
}
return ipfsPath, ipfsPath.IsValid()
}
func readIpfsConfig(repoRoot *string) (bootstrap []string, peers []peer.AddrInfo) {
if repoRoot == nil {
return
}
cfgPath, err := config.Filename(*repoRoot)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
cfgFile, err := os.Open(cfgPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
defer cfgFile.Close()
// Attempt to read bootstrap addresses
var bootstrapCfg struct {
Bootstrap []string
}
err = json.NewDecoder(cfgFile).Decode(&bootstrapCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read bootstrap peers from config")
} else {
bootstrap = bootstrapCfg.Bootstrap
}
if _, err = cfgFile.Seek(0, 0); err != nil {
// If Seek fails, only log the error and continue on to try to read the
// peering config anyway as it might still be readable
fmt.Fprintln(os.Stderr, err)
}
// Attempt to read peers
var peeringCfg struct {
Peering config.Peering
}
err = json.NewDecoder(cfgFile).Decode(&peeringCfg)
if err != nil {
fmt.Fprintln(os.Stderr, "cannot read peering from config")
} else {
peers = peeringCfg.Peering.Peers
}
return
}

144
repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go
View File

@ -25,7 +25,7 @@ func TestIpfsFetcher(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
fetcher := NewIpfsFetcher("", 0, nil, nil)
fetcher := NewIpfsFetcher("", 0, nil)
defer fetcher.Close()
rc, err := fetcher.Fetch(ctx, "go-ipfs/versions")
@ -63,11 +63,11 @@ func TestInitIpfsFetcher(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := NewIpfsFetcher("", 0, nil, nil)
f := NewIpfsFetcher("", 0, nil)
defer f.Close()
// Init ipfs repo
f.ipfsTmpDir, f.openErr = initTempNode(ctx, f.bootstrap, f.peers)
f.ipfsTmpDir, f.openErr = initTempNode(ctx, nil, nil)
if f.openErr != nil {
t.Fatalf("failed to initialize ipfs node: %s", f.openErr)
}
@ -110,6 +110,144 @@ func TestInitIpfsFetcher(t *testing.T) {
}
}
func TestReadIpfsConfig(t *testing.T) {
var testConfig = `
{
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"],
"Keep": "cache"
},
"Peering": {
"Peers": [
{
"ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5",
"Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"]
}
]
}
}
`
noSuchDir := "no_such_dir-5953aa51-1145-4efd-afd1-a069075fcf76"
bootstrap, peers := readIpfsConfig(&noSuchDir)
if bootstrap != nil {
t.Error("expected nil bootstrap")
}
if peers != nil {
t.Error("expected nil peers")
}
tmpDir := makeConfig(t, testConfig)
bootstrap, peers = readIpfsConfig(nil)
if bootstrap != nil || peers != nil {
t.Fatal("expected nil ipfs config items")
}
bootstrap, peers = readIpfsConfig(&tmpDir)
if len(bootstrap) != 2 {
t.Fatal("wrong number of bootstrap addresses")
}
if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" {
t.Fatal("wrong bootstrap address")
}
if len(peers) != 1 {
t.Fatal("wrong number of peers")
}
peer := peers[0]
if peer.ID.String() != "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5" {
t.Errorf("wrong ID for first peer")
}
if len(peer.Addrs) != 2 {
t.Error("wrong number of addrs for first peer")
}
}
func TestBadBootstrappingIpfsConfig(t *testing.T) {
const configBadBootstrap = `
{
"Bootstrap": "unreadable",
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1"],
"Keep": "cache"
},
"Peering": {
"Peers": [
{
"ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5",
"Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"]
}
]
}
}
`
tmpDir := makeConfig(t, configBadBootstrap)
bootstrap, peers := readIpfsConfig(&tmpDir)
if bootstrap != nil {
t.Fatal("expected nil bootstrap")
}
if len(peers) != 1 {
t.Fatal("wrong number of peers")
}
if len(peers[0].Addrs) != 2 {
t.Error("wrong number of addrs for first peer")
}
os.RemoveAll(tmpDir)
}
func TestBadPeersIpfsConfig(t *testing.T) {
const configBadPeers = `
{
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1"],
"Keep": "cache"
},
"Peering": "Unreadable-data"
}
`
tmpDir := makeConfig(t, configBadPeers)
bootstrap, peers := readIpfsConfig(&tmpDir)
if peers != nil {
t.Fatal("expected nil peers")
}
if len(bootstrap) != 2 {
t.Fatal("wrong number of bootstrap addresses")
}
if bootstrap[0] != "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" {
t.Fatal("wrong bootstrap address")
}
}
func makeConfig(t *testing.T, configData string) string {
tmpDir := t.TempDir()
cfgFile, err := os.Create(filepath.Join(tmpDir, "config"))
if err != nil {
t.Fatal(err)
}
if _, err = cfgFile.Write([]byte(configData)); err != nil {
t.Fatal(err)
}
if err = cfgFile.Close(); err != nil {
t.Fatal(err)
}
return tmpDir
}
func skipUnlessEpic(t *testing.T) {
if os.Getenv("IPFS_EPIC_TEST") == "" {
t.SkipNow()

88
repo/fsrepo/migrations/migrations.go
View File

@ -2,15 +2,20 @@ package migrations
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"os/exec"
"path"
"runtime"
"strings"
"sync"
config "github.com/ipfs/go-ipfs-config"
)
const (
@ -107,6 +112,89 @@ func ExeName(name string) string {
return name
}
// ReadMigrationConfig reads the Migration section of the IPFS config, avoiding
// reading anything other than the Migration section. That way, we're free to
// make arbitrary changes to all _other_ sections in migrations.
func ReadMigrationConfig(repoRoot string) (*config.Migration, error) {
var cfg struct {
Migration config.Migration
}
cfgPath, err := config.Filename(repoRoot)
if err != nil {
return nil, err
}
cfgFile, err := os.Open(cfgPath)
if err != nil {
return nil, err
}
defer cfgFile.Close()
err = json.NewDecoder(cfgFile).Decode(&cfg)
if err != nil {
return nil, err
}
switch cfg.Migration.Keep {
case "":
cfg.Migration.Keep = config.DefaultMigrationKeep
case "discard", "cache", "keep":
default:
return nil, errors.New("unknown config value, Migrations.Keep must be 'cache', 'pin', or 'discard'")
}
if len(cfg.Migration.DownloadSources) == 0 {
cfg.Migration.DownloadSources = config.DefaultMigrationDownloadSources
}
return &cfg.Migration, nil
}
// GetMigrationFetcher creates one or more fetchers according to
// downloadSources,
func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetcher func(string) Fetcher) (Fetcher, error) {
const httpUserAgent = "go-ipfs"
var fetchers []Fetcher
for _, src := range downloadSources {
src := strings.TrimSpace(src)
switch src {
case "HTTPS", "https", "HTTP", "http":
fetchers = append(fetchers, NewHttpFetcher(distPath, "", httpUserAgent, 0))
case "IPFS", "ipfs":
if newIpfsFetcher != nil {
fetchers = append(fetchers, newIpfsFetcher(distPath))
}
default:
u, err := url.Parse(src)
if err != nil {
return nil, fmt.Errorf("bad gateway address: %s", err)
}
switch u.Scheme {
case "":
u.Scheme = "https"
case "https", "http":
default:
return nil, errors.New("bad gateway address: url scheme must be http or https")
}
fetchers = append(fetchers, NewHttpFetcher(distPath, u.String(), httpUserAgent, 0))
case "":
// Ignore empty string
}
}
switch len(fetchers) {
case 0:
return nil, errors.New("no sources specified")
case 1:
return fetchers[0], nil
}
// Wrap fetchers in a MultiFetcher to try them in order
return NewMultiFetcher(fetchers...), nil
}
func migrationName(from, to int) string {
return fmt.Sprintf("fs-repo-%d-to-%d", from, to)
}

232
repo/fsrepo/migrations/migrations_test.go
View File

@ -3,20 +3,18 @@ package migrations
import (
"context"
"fmt"
"io/ioutil"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
config "github.com/ipfs/go-ipfs-config"
)
func TestFindMigrations(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "migratetest")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -63,11 +61,7 @@ func TestFindMigrations(t *testing.T) {
}
func TestFindMigrationsReverse(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "migratetest")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -121,11 +115,7 @@ func TestFetchMigrations(t *testing.T) {
defer ts.Close()
fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
tmpDir, err := ioutil.TempDir("", "migratetest")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
needed := []string{"fs-repo-1-to-2", "fs-repo-2-to-3"}
buf := new(strings.Builder)
@ -157,16 +147,12 @@ func TestFetchMigrations(t *testing.T) {
}
func TestRunMigrations(t *testing.T) {
fakeHome, err := ioutil.TempDir("", "testhome")
if err != nil {
panic(err)
}
defer os.RemoveAll(fakeHome)
fakeHome := t.TempDir()
os.Setenv("HOME", fakeHome)
fakeIpfs := filepath.Join(fakeHome, ".ipfs")
err = os.Mkdir(fakeIpfs, os.ModePerm)
err := os.Mkdir(fakeIpfs, os.ModePerm)
if err != nil {
panic(err)
}
@ -211,3 +197,205 @@ func createFakeBin(from, to int, tmpDir string) {
panic(err)
}
}
var testConfig = `
{
"Bootstrap": [
"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
],
"Migration": {
"DownloadSources": ["IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"],
"Keep": "cache"
},
"Peering": {
"Peers": [
{
"ID": "12D3KooWGC6TvWhfapngX6wvJHMYvKpDMXPb3ZnCZ6dMoaMtimQ5",
"Addrs": ["/ip4/127.0.0.1/tcp/4001", "/ip4/127.0.0.1/udp/4001/quic"]
}
]
}
}
`
func TestReadMigrationConfigDefaults(t *testing.T) {
tmpDir := makeConfig(t, "{}")
cfg, err := ReadMigrationConfig(tmpDir)
if err != nil {
t.Fatal(err)
}
if cfg.Keep != config.DefaultMigrationKeep {
t.Error("expected default value for Keep")
}
if len(cfg.DownloadSources) != len(config.DefaultMigrationDownloadSources) {
t.Fatal("expected default number of download sources")
}
for i, src := range config.DefaultMigrationDownloadSources {
if cfg.DownloadSources[i] != src {
t.Errorf("wrong DownloadSource: %s", cfg.DownloadSources[i])
}
}
}
func TestReadMigrationConfigErrors(t *testing.T) {
tmpDir := makeConfig(t, `{"Migration": {"Keep": "badvalue"}}`)
_, err := ReadMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
if !strings.HasPrefix(err.Error(), "unknown") {
t.Fatal("did not get expected error:", err)
}
os.RemoveAll(tmpDir)
_, err = ReadMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
tmpDir = makeConfig(t, `}{`)
_, err = ReadMigrationConfig(tmpDir)
if err == nil {
t.Fatal("expected error")
}
}
func TestReadMigrationConfig(t *testing.T) {
tmpDir := makeConfig(t, testConfig)
cfg, err := ReadMigrationConfig(tmpDir)
if err != nil {
t.Fatal(err)
}
if len(cfg.DownloadSources) != 4 {
t.Fatal("wrong number of DownloadSources")
}
expect := []string{"IPFS", "HTTP", "127.0.0.1", "https://127.0.1.1"}
for i := range expect {
if cfg.DownloadSources[i] != expect[i] {
t.Errorf("wrong DownloadSource at %d", i)
}
}
if cfg.Keep != "cache" {
t.Error("wrong value for Keep")
}
}
type mockIpfsFetcher struct{}
func (m *mockIpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) {
return nil, nil
}
func (m *mockIpfsFetcher) Close() error {
return nil
}
func TestGetMigrationFetcher(t *testing.T) {
var f Fetcher
var err error
newIpfsFetcher := func(distPath string) Fetcher {
return &mockIpfsFetcher{}
}
downloadSources := []string{"ftp://bad.gateway.io"}
_, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") {
t.Fatal("Expected bad gateway address error, got:", err)
}
downloadSources = []string{"::bad.gateway.io"}
_, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err == nil || !strings.HasPrefix(err.Error(), "bad gateway addr") {
t.Fatal("Expected bad gateway address error, got:", err)
}
downloadSources = []string{"http://localhost"}
f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*HttpFetcher); !ok {
t.Fatal("expected HttpFetcher")
}
downloadSources = []string{"ipfs"}
f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*mockIpfsFetcher); !ok {
t.Fatal("expected IpfsFetcher")
}
downloadSources = []string{"http"}
f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err != nil {
t.Fatal(err)
}
if _, ok := f.(*HttpFetcher); !ok {
t.Fatal("expected HttpFetcher")
}
downloadSources = []string{"IPFS", "HTTPS"}
f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err != nil {
t.Fatal(err)
}
mf, ok := f.(*MultiFetcher)
if !ok {
t.Fatal("expected MultiFetcher")
}
if mf.Len() != 2 {
t.Fatal("expected 2 fetchers in MultiFetcher")
}
downloadSources = []string{"ipfs", "https", "some.domain.io"}
f, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err != nil {
t.Fatal(err)
}
mf, ok = f.(*MultiFetcher)
if !ok {
t.Fatal("expected MultiFetcher")
}
if mf.Len() != 3 {
t.Fatal("expected 3 fetchers in MultiFetcher")
}
downloadSources = nil
_, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err == nil {
t.Fatal("expected error when no sources specified")
}
downloadSources = []string{"", ""}
_, err = GetMigrationFetcher(downloadSources, "", newIpfsFetcher)
if err == nil {
t.Fatal("expected error when empty string fetchers specified")
}
}
func makeConfig(t *testing.T, configData string) string {
tmpDir := t.TempDir()
cfgFile, err := os.Create(filepath.Join(tmpDir, "config"))
if err != nil {
t.Fatal(err)
}
if _, err = cfgFile.Write([]byte(configData)); err != nil {
t.Fatal(err)
}
if err = cfgFile.Close(); err != nil {
t.Fatal(err)
}
return tmpDir
}

16
repo/fsrepo/migrations/unpack_test.go
View File

@ -33,14 +33,10 @@ func TestUnpackArchive(t *testing.T) {
}
func TestUnpackTgz(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "testunpacktgz")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
badTarGzip := filepath.Join(tmpDir, "bad.tar.gz")
err = ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644)
err := ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644)
if err != nil {
panic(err)
}
@ -81,14 +77,10 @@ func TestUnpackTgz(t *testing.T) {
}
func TestUnpackZip(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "testunpackzip")
if err != nil {
panic(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
badZip := filepath.Join(tmpDir, "bad.zip")
err = ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644)
err := ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644)
if err != nil {
panic(err)
}