mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
repo/fsrepo/migrations: verified HTTP migrations (#10324)
This commit is contained in:
Henrique Dias
GitHub
parent
1514785074
commit
595e1ba268
@ -10,6 +10,7 @@
|
||||
- [IPNS resolver cache's TTL can now be configured](#ipns-resolver-caches-ttl-can-now-be-configured)
|
||||
- [RPC client: deprecated DHT API, added Routing API](#rpc-client-deprecated-dht-api-added-routing-api)
|
||||
- [Deprecated DHT commands removed from `/api/v0/dht`](#deprecated-dht-commands-removed-from-apiv0dht)
|
||||
- [Repository migrations are now trustless](#repository-migrations-are-now-trustless)
|
||||
- [📝 Changelog](#-changelog)
|
||||
- [👨👩👧👦 Contributors](#-contributors)
|
||||
|
||||
@ -37,6 +38,10 @@ In the next version, all DHT deprecated methods will be removed from the Go RPC
|
||||
|
||||
All the DHT commands that were deprecated for over a year were finally removed from `/api/v0/dht`. Users should switch to modern `/api/v0/routing` which works with [both Amino DHT and Delegated Routers](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing).
|
||||
|
||||
#### Repository migrations are now trustless
|
||||
|
||||
Kubo now only uses [trustless requests](https://specs.ipfs.tech/http-gateways/trustless-gateway/) (e.g., CAR files) when downloading repository migrations via HTTP. This further strengthens Kubo by not delegating trust to public gateways. The migration binaries are locally verified before being executed.
|
||||
|
||||
### 📝 Changelog
|
||||
|
||||
### 👨👩👧👦 Contributors
|
||||
|
||||
@ -12,14 +12,18 @@ Kubo's Gateway implementation follows [ipfs/specs: Specification for HTTP Gatewa
|
||||
|
||||
By default, Kubo nodes run
|
||||
a [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/`
|
||||
and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`
|
||||
and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`.
|
||||
Both support [trustless responses](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) as opt-in via `Accept` header.
|
||||
|
||||
Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file.
|
||||
|
||||
### Public gateways
|
||||
|
||||
Protocol Labs provides a public gateway at `https://ipfs.io` (path) and `https://dweb.link` (subdomain).
|
||||
If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our* gateway.
|
||||
Protocol Labs provides a public gateway at
|
||||
`https://ipfs.io` ([path](https://specs.ipfs.tech/http-gateways/path-gateway/)),
|
||||
`https://dweb.link` ([subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)),
|
||||
and `https://trustless-gateway.link` ([trustless](https://specs.ipfs.tech/http-gateways/trustless-gateway/) only).
|
||||
If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from a *public goods* gateway.
|
||||
|
||||
There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/
|
||||
|
||||
@ -105,7 +109,7 @@ This is a rough equivalent of `ipfs dag export`.
|
||||
|
||||
## Deprecated Subset of RPC API
|
||||
|
||||
For legacy reasons, the gateway port exposes a small subset of RPC API under `/api/v0/`.
|
||||
For legacy reasons, some gateways may expose a small subset of RPC API under `/api/v0/`.
|
||||
While this read-only API exposes a read-only, "safe" subset of the normal API,
|
||||
it is deprecated and should not be used for greenfield projects.
|
||||
|
||||
|
||||
1
go.mod
1
go.mod
@ -136,6 +136,7 @@ require (
|
||||
github.com/ipfs/go-bitfield v1.1.0 // indirect
|
||||
github.com/ipfs/go-blockservice v0.5.0 // indirect
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
|
||||
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
|
||||
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
|
||||
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
|
||||
|
||||
3
go.sum
3
go.sum
@ -330,11 +330,13 @@ github.com/ipfs/boxo v0.17.1-0.20240206084652-79cb4e2886d7/go.mod h1:pIZgTWdm3k3
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
|
||||
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
|
||||
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
|
||||
github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY=
|
||||
github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w=
|
||||
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
|
||||
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
@ -367,6 +369,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
|
||||
github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
|
||||
github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
|
||||
github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
|
||||
github.com/ipfs/go-ipfs-cmds v0.10.0 h1:ZB4+RgYaH4UARfJY0uLKl5UXgApqnRjKbuCiJVcErYk=
|
||||
github.com/ipfs/go-ipfs-cmds v0.10.0/go.mod h1:sX5d7jkCft9XLPnkgEfXY0z2UBOB5g6fh/obBS0enJE=
|
||||
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
|
||||
|
||||
@ -5,56 +5,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func createTestServer() *httptest.Server {
|
||||
reqHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||
defer r.Body.Close()
|
||||
if strings.Contains(r.URL.Path, "not-here") {
|
||||
http.NotFound(w, r)
|
||||
} else if strings.HasSuffix(r.URL.Path, "versions") {
|
||||
fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n")
|
||||
} else if strings.HasSuffix(r.URL.Path, ".tar.gz") {
|
||||
createFakeArchive(r.URL.Path, false, w)
|
||||
} else if strings.HasSuffix(r.URL.Path, "zip") {
|
||||
createFakeArchive(r.URL.Path, true, w)
|
||||
} else {
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
return httptest.NewServer(http.HandlerFunc(reqHandler))
|
||||
}
|
||||
|
||||
func createFakeArchive(name string, archZip bool, w io.Writer) {
|
||||
fileName := strings.Split(path.Base(name), "_")[0]
|
||||
root := path.Base(path.Dir(path.Dir(name)))
|
||||
|
||||
// Simulate fetching go-ipfs, which has "ipfs" as the name in the archive.
|
||||
if fileName == "go-ipfs" {
|
||||
fileName = "ipfs"
|
||||
}
|
||||
fileName = ExeName(fileName)
|
||||
|
||||
var err error
|
||||
if archZip {
|
||||
err = writeZip(root, fileName, "FAKE DATA", w)
|
||||
} else {
|
||||
err = writeTarGzip(root, fileName, "FAKE DATA", w)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDistPath(t *testing.T) {
|
||||
os.Unsetenv(envIpfsDistPath)
|
||||
distPath := GetDistPathEnv("")
|
||||
@ -91,12 +48,9 @@ func TestHttpFetch(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
fetcher := NewHttpFetcher("", ts.URL, "", 0)
|
||||
|
||||
out, err := fetcher.Fetch(ctx, "/versions")
|
||||
out, err := fetcher.Fetch(ctx, "/kubo/versions")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -120,7 +74,7 @@ func TestHttpFetch(t *testing.T) {
|
||||
|
||||
// Check not found
|
||||
_, err = fetcher.Fetch(ctx, "/no_such_file")
|
||||
if err == nil || !strings.Contains(err.Error(), "404") {
|
||||
if err == nil || !strings.Contains(err.Error(), "no link") {
|
||||
t.Fatal("expected error 404")
|
||||
}
|
||||
}
|
||||
@ -131,10 +85,7 @@ func TestFetchBinary(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
|
||||
fetcher := NewHttpFetcher("", ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
vers, err := DistVersions(ctx, fetcher, distFSRM, false)
|
||||
if err != nil {
|
||||
@ -154,7 +105,7 @@ func TestFetchBinary(t *testing.T) {
|
||||
|
||||
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
|
||||
|
||||
bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
|
||||
bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -167,12 +118,12 @@ func TestFetchBinary(t *testing.T) {
|
||||
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
|
||||
|
||||
// Check error is destination already exists and is not directory
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin)
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin)
|
||||
if !os.IsExist(err) {
|
||||
t.Fatal("expected 'exists' error, got", err)
|
||||
}
|
||||
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
|
||||
if !os.IsExist(err) {
|
||||
t.Error("expected 'exists' error, got:", err)
|
||||
}
|
||||
@ -192,7 +143,7 @@ func TestFetchBinary(t *testing.T) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
|
||||
if !os.IsPermission(err) {
|
||||
t.Error("expected 'permission' error, got:", err)
|
||||
}
|
||||
@ -207,13 +158,13 @@ func TestFetchBinary(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check error if failure to fetch due to bad dist
|
||||
_, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir)
|
||||
if err == nil || !strings.Contains(err.Error(), "Not Found") {
|
||||
_, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir)
|
||||
if err == nil || !strings.Contains(err.Error(), "no link") {
|
||||
t.Error("expected 'Not Found' error, got:", err)
|
||||
}
|
||||
|
||||
// Check error if failure to unpack archive
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir)
|
||||
_, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir)
|
||||
if err == nil || err.Error() != "no binary found in archive" {
|
||||
t.Error("expected 'no binary found in archive' error")
|
||||
}
|
||||
@ -223,15 +174,12 @@ func TestMultiFetcher(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
|
||||
badFetcher := NewHttpFetcher("", "bad-url", "", 0)
|
||||
fetcher := NewHttpFetcher("", ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
mf := NewMultiFetcher(badFetcher, fetcher)
|
||||
|
||||
vers, err := mf.Fetch(ctx, "/versions")
|
||||
vers, err := mf.Fetch(ctx, "/kubo/versions")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -2,21 +2,40 @@ package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
gopath "path"
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/boxo/blockservice"
|
||||
"github.com/ipfs/boxo/blockstore"
|
||||
"github.com/ipfs/boxo/exchange/offline"
|
||||
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
|
||||
files "github.com/ipfs/boxo/files"
|
||||
"github.com/ipfs/boxo/ipld/merkledag"
|
||||
unixfile "github.com/ipfs/boxo/ipld/unixfs/file"
|
||||
"github.com/ipfs/boxo/ipns"
|
||||
"github.com/ipfs/boxo/namesys"
|
||||
"github.com/ipfs/boxo/path"
|
||||
"github.com/ipfs/boxo/path/resolver"
|
||||
"github.com/ipfs/go-datastore"
|
||||
dssync "github.com/ipfs/go-datastore/sync"
|
||||
"github.com/ipfs/go-unixfsnode"
|
||||
gocarv2 "github.com/ipld/go-car/v2"
|
||||
dagpb "github.com/ipld/go-codec-dagpb"
|
||||
madns "github.com/multiformats/go-multiaddr-dns"
|
||||
)
|
||||
|
||||
const (
|
||||
// default is different name than ipfs.io which is being blocked by some ISPs
|
||||
defaultGatewayURL = "https://dweb.link"
|
||||
defaultGatewayURL = "https://trustless-gateway.link"
|
||||
// Default maximum download size.
|
||||
defaultFetchLimit = 1024 * 1024 * 512
|
||||
)
|
||||
|
||||
// HttpFetcher fetches files over HTTP.
|
||||
// HttpFetcher fetches files over HTTP using verifiable CAR archives.
|
||||
type HttpFetcher struct { //nolint
|
||||
distPath string
|
||||
gateway string
|
||||
@ -26,7 +45,7 @@ type HttpFetcher struct { //nolint
|
||||
|
||||
var _ Fetcher = (*HttpFetcher)(nil)
|
||||
|
||||
// NewHttpFetcher creates a new HttpFetcher
|
||||
// NewHttpFetcher creates a new [HttpFetcher].
|
||||
//
|
||||
// Specifying "" for distPath sets the default IPNS path.
|
||||
// Specifying "" for gateway sets the default.
|
||||
@ -62,13 +81,89 @@ func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *Http
|
||||
// Fetch attempts to fetch the file at the given path, from the distribution
|
||||
// site configured for this HttpFetcher.
|
||||
func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) {
|
||||
gwURL := f.gateway + path.Join(f.distPath, filePath)
|
||||
fmt.Printf("Fetching with HTTP: %q\n", gwURL)
|
||||
imPath, err := f.resolvePath(ctx, gopath.Join(f.distPath, filePath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("path could not be resolved: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, gwURL, nil)
|
||||
rc, err := f.httpRequest(ctx, imPath, "application/vnd.ipld.car")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch CAR: %w", err)
|
||||
}
|
||||
|
||||
return carStreamToFileBytes(ctx, rc, imPath)
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) resolvePath(ctx context.Context, pathStr string) (path.ImmutablePath, error) {
|
||||
p, err := path.NewPath(pathStr)
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, fmt.Errorf("path is invalid: %w", err)
|
||||
}
|
||||
|
||||
for p.Mutable() {
|
||||
// Download IPNS record and verify through the gateway, or resolve the
|
||||
// DNSLink with the default DNS resolver.
|
||||
name, err := ipns.NameFromString(p.Segments()[1])
|
||||
if err == nil {
|
||||
p, err = f.resolveIPNS(ctx, name)
|
||||
} else {
|
||||
p, err = f.resolveDNSLink(ctx, p)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return path.NewImmutablePath(p)
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) resolveIPNS(ctx context.Context, name ipns.Name) (path.Path, error) {
|
||||
rc, err := f.httpRequest(ctx, name.AsPath(), "application/vnd.ipfs.ipns-record")
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, err
|
||||
}
|
||||
|
||||
rc = NewLimitReadCloser(rc, int64(ipns.MaxRecordSize))
|
||||
rawRecord, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, err
|
||||
}
|
||||
|
||||
rec, err := ipns.UnmarshalRecord(rawRecord)
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, err
|
||||
}
|
||||
|
||||
err = ipns.ValidateWithName(rec, name)
|
||||
if err != nil {
|
||||
return path.ImmutablePath{}, err
|
||||
}
|
||||
|
||||
return rec.Value()
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) resolveDNSLink(ctx context.Context, p path.Path) (path.Path, error) {
|
||||
dnsResolver := namesys.NewDNSResolver(madns.DefaultResolver.LookupTXT)
|
||||
res, err := dnsResolver.Resolve(ctx, p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res.Path, nil
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) httpRequest(ctx context.Context, p path.Path, accept string) (io.ReadCloser, error) {
|
||||
url := f.gateway + p.String()
|
||||
fmt.Printf("Fetching with HTTP: %q\n", url)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http.NewRequest error: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", accept)
|
||||
|
||||
if f.userAgent != "" {
|
||||
req.Header.Set("User-Agent", f.userAgent)
|
||||
@ -85,7 +180,7 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading error body: %w", err)
|
||||
}
|
||||
return nil, fmt.Errorf("GET %s error: %s: %s", gwURL, resp.Status, string(mes))
|
||||
return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes))
|
||||
}
|
||||
|
||||
var rc io.ReadCloser
|
||||
@ -94,11 +189,69 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error
|
||||
} else {
|
||||
rc = resp.Body
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
return io.ReadAll(rc)
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
func (f *HttpFetcher) Close() error {
|
||||
return nil
|
||||
func carStreamToFileBytes(ctx context.Context, r io.ReadCloser, imPath path.ImmutablePath) ([]byte, error) {
|
||||
defer r.Close()
|
||||
|
||||
// Create temporary block datastore and dag service.
|
||||
dataStore := dssync.MutexWrap(datastore.NewMapDatastore())
|
||||
blockStore := blockstore.NewBlockstore(dataStore)
|
||||
blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
|
||||
dagService := merkledag.NewDAGService(blockService)
|
||||
|
||||
defer dagService.Blocks.Close()
|
||||
defer dataStore.Close()
|
||||
|
||||
// Create CAR reader
|
||||
car, err := gocarv2.NewBlockReader(r)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return nil, fmt.Errorf("error creating car reader: %s", err)
|
||||
}
|
||||
|
||||
// Add all blocks to the blockstore.
|
||||
for {
|
||||
block, err := car.Next()
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, fmt.Errorf("error reading block from car: %s", err)
|
||||
} else if block == nil {
|
||||
break
|
||||
}
|
||||
|
||||
err = blockStore.Put(ctx, block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error putting block in blockstore: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
fetcherCfg := bsfetcher.NewFetcherConfig(blockService)
|
||||
fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
|
||||
fetcher := fetcherCfg.WithReifier(unixfsnode.Reify)
|
||||
resolver := resolver.NewBasicResolver(fetcher)
|
||||
|
||||
cid, _, err := resolver.ResolveToLastNode(ctx, imPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve: %w", err)
|
||||
}
|
||||
|
||||
nd, err := dagService.Get(ctx, cid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve: %w", err)
|
||||
}
|
||||
|
||||
// Make UnixFS file out of the node.
|
||||
uf, err := unixfile.NewUnixfsFile(ctx, dagService, nd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building unixfs file: %s", err)
|
||||
}
|
||||
|
||||
// Check if it's a file and return.
|
||||
if f, ok := uf.(files.File); ok {
|
||||
return io.ReadAll(f)
|
||||
}
|
||||
|
||||
return nil, errors.New("unexpected unixfs node type")
|
||||
}
|
||||
|
||||
@ -110,9 +110,7 @@ func TestFetchMigrations(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
@ -162,9 +160,7 @@ func TestRunMigrations(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
233
repo/fsrepo/migrations/setup_test.go
Normal file
233
repo/fsrepo/migrations/setup_test.go
Normal file
@ -0,0 +1,233 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/boxo/blockservice"
|
||||
"github.com/ipfs/boxo/exchange/offline"
|
||||
"github.com/ipfs/boxo/gateway"
|
||||
blocks "github.com/ipfs/go-block-format"
|
||||
"github.com/ipfs/go-cid"
|
||||
"github.com/ipfs/go-unixfsnode/data/builder"
|
||||
"github.com/ipld/go-car/v2"
|
||||
carblockstore "github.com/ipld/go-car/v2/blockstore"
|
||||
"github.com/ipld/go-ipld-prime"
|
||||
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
|
||||
"github.com/multiformats/go-multicodec"
|
||||
"github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
var (
|
||||
testIpfsDist string
|
||||
testServer *httptest.Server
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Setup test data
|
||||
testDataDir := makeTestData()
|
||||
defer os.RemoveAll(testDataDir)
|
||||
|
||||
testCar := makeTestCar(testDataDir)
|
||||
defer os.RemoveAll(testCar)
|
||||
|
||||
// Setup test gateway
|
||||
fd := setupTestGateway(testCar)
|
||||
defer fd.Close()
|
||||
|
||||
// Run tests
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func makeTestData() string {
|
||||
tempDir, err := os.MkdirTemp("", "kubo-migrations-test-*")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
versions := []string{"v1.0.0", "v1.1.0", "v1.1.2", "v2.0.0-rc1", "2.0.0", "v2.0.1"}
|
||||
packages := []string{"kubo", "go-ipfs", "fs-repo-migrations", "fs-repo-1-to-2", "fs-repo-2-to-3", "fs-repo-9-to-10", "fs-repo-10-to-11"}
|
||||
|
||||
// Generate fake data
|
||||
for _, name := range packages {
|
||||
err = os.MkdirAll(filepath.Join(tempDir, name), 0777)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(filepath.Join(tempDir, name, "versions"), []byte(strings.Join(versions, "\n")+"\n"), 0666)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, version := range versions {
|
||||
filename, archName := makeArchivePath(name, name, version, "tar.gz")
|
||||
createFakeArchive(filepath.Join(tempDir, filename), archName, false)
|
||||
|
||||
filename, archName = makeArchivePath(name, name, version, "zip")
|
||||
createFakeArchive(filepath.Join(tempDir, filename), archName, true)
|
||||
}
|
||||
}
|
||||
|
||||
return tempDir
|
||||
}
|
||||
|
||||
func createFakeArchive(archName, name string, archZip bool) {
|
||||
err := os.MkdirAll(filepath.Dir(archName), 0777)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
fileName := strings.Split(path.Base(name), "_")[0]
|
||||
root := fileName
|
||||
|
||||
// Simulate fetching go-ipfs, which has "ipfs" as the name in the archive.
|
||||
if fileName == "go-ipfs" || fileName == "kubo" {
|
||||
fileName = "ipfs"
|
||||
}
|
||||
fileName = ExeName(fileName)
|
||||
|
||||
if archZip {
|
||||
err = writeZipFile(archName, root, fileName, "FAKE DATA")
|
||||
} else {
|
||||
err = writeTarGzipFile(archName, root, fileName, "FAKE DATA")
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// makeTestCar makes a CAR file with the directory [testData]. This code is mostly
|
||||
// sourced from https://github.com/ipld/go-car/blob/1e2f0bd2c44ee31f48a8f602b25b5671cc0c4687/cmd/car/create.go
|
||||
func makeTestCar(testData string) string {
|
||||
// make a cid with the right length that we eventually will patch with the root.
|
||||
hasher, err := multihash.GetHasher(multihash.SHA2_256)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
digest := hasher.Sum([]byte{})
|
||||
hash, err := multihash.Encode(digest, multihash.SHA2_256)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash)
|
||||
|
||||
// Make CAR file
|
||||
fd, err := os.CreateTemp("", "kubo-migrations-test-*.car")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer fd.Close()
|
||||
filename := fd.Name()
|
||||
|
||||
rw, err := carblockstore.OpenReadWriteFile(fd, []cid.Cid{proxyRoot}, carblockstore.WriteAsCarV1(true))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rw.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
ls := cidlink.DefaultLinkSystem()
|
||||
ls.TrustedStorage = true
|
||||
ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) {
|
||||
cl, ok := l.(cidlink.Link)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not a cidlink")
|
||||
}
|
||||
blk, err := rw.Get(ctx, cl.Cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.NewBuffer(blk.RawData()), nil
|
||||
}
|
||||
ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
|
||||
buf := bytes.NewBuffer(nil)
|
||||
return buf, func(l ipld.Link) error {
|
||||
cl, ok := l.(cidlink.Link)
|
||||
if !ok {
|
||||
return fmt.Errorf("not a cidlink")
|
||||
}
|
||||
blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return rw.Put(ctx, blk)
|
||||
}, nil
|
||||
}
|
||||
|
||||
l, _, err := builder.BuildUnixFSRecursive(testData, &ls)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
rcl, ok := l.(cidlink.Link)
|
||||
if !ok {
|
||||
panic(fmt.Errorf("could not interpret %s", l))
|
||||
}
|
||||
|
||||
if err := rw.Finalize(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// re-open/finalize with the final root.
|
||||
err = car.ReplaceRootsInFile(filename, []cid.Cid{rcl.Cid})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return filename
|
||||
}
|
||||
|
||||
func setupTestGateway(testCar string) io.Closer {
|
||||
blockService, roots, fd, err := newBlockServiceFromCAR(testCar)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if len(roots) != 1 {
|
||||
panic("expected car with 1 root")
|
||||
}
|
||||
|
||||
backend, err := gateway.NewBlocksBackend(blockService)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
conf := gateway.Config{
|
||||
NoDNSLink: false,
|
||||
DeserializedResponses: false,
|
||||
}
|
||||
|
||||
testIpfsDist = "/ipfs/" + roots[0].String()
|
||||
testServer = httptest.NewServer(gateway.NewHandler(conf, backend))
|
||||
|
||||
return fd
|
||||
}
|
||||
|
||||
func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) {
|
||||
r, err := os.Open(filepath)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
bs, err := carblockstore.NewReadOnly(r, nil)
|
||||
if err != nil {
|
||||
_ = r.Close()
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
roots, err := bs.Roots()
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
blockService := blockservice.New(bs, offline.Exchange(bs))
|
||||
return blockService, roots, r, nil
|
||||
}
|
||||
@ -13,9 +13,7 @@ func TestDistVersions(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
fetcher := NewHttpFetcher("", ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
vers, err := DistVersions(ctx, fetcher, testDist, true)
|
||||
if err != nil {
|
||||
@ -32,9 +30,7 @@ func TestLatestDistVersion(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
ts := createTestServer()
|
||||
defer ts.Close()
|
||||
fetcher := NewHttpFetcher("", ts.URL, "", 0)
|
||||
fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
|
||||
|
||||
latest, err := LatestDistVersion(ctx, fetcher, testDist, false)
|
||||
if err != nil {
|
||||
|
||||
@ -36,15 +36,20 @@ test_expect_success "configure migration sources" '
|
||||
ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]"
|
||||
'
|
||||
|
||||
test_expect_success "make repo be version 4" '
|
||||
echo 4 > "$IPFS_PATH/version"
|
||||
test_expect_success "setup http response" '
|
||||
mkdir migration &&
|
||||
echo "v1.1.1" > migration/versions &&
|
||||
mkdir -p migration/fs-repo-6-to-7 &&
|
||||
echo "v1.1.1" > migration/fs-repo-6-to-7/versions &&
|
||||
CID=$(ipfs add -r -Q migration) &&
|
||||
echo "HTTP/1.1 200 OK" > vers_resp &&
|
||||
echo "Content-Type: application/vnd.ipld.car" >> vers_resp &&
|
||||
echo "" >> vers_resp &&
|
||||
ipfs dag export $CID >> vers_resp
|
||||
'
|
||||
|
||||
test_expect_success "setup http response" '
|
||||
echo "HTTP/1.1 200 OK" > vers_resp &&
|
||||
echo "Content-Length: 7" >> vers_resp &&
|
||||
echo "" >> vers_resp &&
|
||||
echo "v1.1.1" >> vers_resp
|
||||
test_expect_success "make repo be version 4" '
|
||||
echo 4 > "$IPFS_PATH/version"
|
||||
'
|
||||
|
||||
test_expect_success "startup fake dists server" '
|
||||
@ -53,7 +58,7 @@ test_expect_success "startup fake dists server" '
|
||||
'
|
||||
|
||||
test_expect_success "docker image runs" '
|
||||
DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG")
|
||||
DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs -e IPFS_DIST_PATH=/ipfs/$CID --net=host "$IMAGE_TAG")
|
||||
'
|
||||
|
||||
test_expect_success "docker container tries to pull migrations from netcat" '
|
||||
|
||||
Loading…
Reference in New Issue
Block a user