test: port gateway sharness tests to Go tests

This commit is contained in:
Gus Eggert 2022-12-14 11:10:48 -05:00
parent d90a9b5b33
commit 5d864faac7
9 changed files with 727 additions and 378 deletions

492
test/cli/gateway_test.go Normal file
View File

@ -0,0 +1,492 @@
package cli
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"regexp"
"testing"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/test/cli/harness"
. "github.com/ipfs/kubo/test/cli/testutils"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr/net"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGateway(t *testing.T) {
t.Parallel()
h := harness.NewT(t)
node := h.NewNode().Init().StartDaemon("--offline")
cid := node.IPFSAddStr("Hello Worlds!")
client := node.GatewayClient()
client.TemplateData = map[string]string{
"CID": cid,
"PeerID": node.PeerID().String(),
}
t.Run("GET IPFS path succeeds", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.CID}}")
assert.Equal(t, 200, resp.StatusCode)
})
t.Run("GET IPFS path with explicit ?filename succeeds with proper header", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.CID}}?filename=testтест.pdf")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t,
`inline; filename="test____.pdf"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.pdf`,
resp.Headers.Get("Content-Disposition"),
)
})
t.Run("GET IPFS path with explicit ?filename and &download=true succeeds with proper header", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.CID}}?filename=testтест.mp4&download=true")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t,
`attachment; filename="test____.mp4"; filename*=UTF-8''test%D1%82%D0%B5%D1%81%D1%82.mp4`,
resp.Headers.Get("Content-Disposition"),
)
})
// https://github.com/ipfs/go-ipfs/issues/4025#issuecomment-342250616
t.Run("GET for Server Worker registration outside of an IPFS content root errors", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.CID}}?filename=sw.js", client.WithHeader("Service-Worker", "script"))
assert.Equal(t, 400, resp.StatusCode)
assert.Contains(t, resp.Body, "navigator.serviceWorker: registration is not allowed for this scope")
})
t.Run("GET IPFS directory path succeeds", func(t *testing.T) {
t.Parallel()
client := node.GatewayClient().DisableRedirects()
pageContents := "hello i am a webpage"
fileContents := "12345"
h.WriteFile("dir/test", fileContents)
h.WriteFile("dir/dirwithindex/index.html", pageContents)
cids := node.IPFS("add", "-r", "-q", filepath.Join(h.Dir, "dir")).Stdout.Lines()
rootCID := cids[len(cids)-1]
client.TemplateData = map[string]string{
"IndexFileCID": cids[0],
"TestFileCID": cids[1],
"RootCID": rootCID,
}
t.Run("GET IPFS the index file CID", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.IndexFileCID}}")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, pageContents, resp.Body)
})
t.Run("GET IPFS the test file CID", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.TestFileCID}}")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, fileContents, resp.Body)
})
t.Run("GET IPFS directory with index.html returns redirect to add trailing slash", func(t *testing.T) {
t.Parallel()
resp := client.Head("/ipfs/{{.RootCID}}/dirwithindex?query=to-remember")
assert.Equal(t, 301, resp.StatusCode)
assert.Equal(t,
fmt.Sprintf("/ipfs/%s/dirwithindex/?query=to-remember", rootCID),
resp.Headers.Get("Location"),
)
})
// This enables go get to parse go-import meta tags from index.html files stored in IPFS
// https://github.com/ipfs/kubo/pull/3963
t.Run("GET IPFS directory with index.html and no trailing slash returns expected output when go-get is passed", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex?go-get=1")
assert.Equal(t, pageContents, resp.Body)
})
t.Run("GET IPFS directory with index.html and trailing slash returns expected output", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.RootCID}}/dirwithindex/?query=to-remember")
assert.Equal(t, pageContents, resp.Body)
})
t.Run("GET IPFS nonexistent file returns 404 (Not Found)", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/{{.RootCID}}/pleaseDontAddMe")
assert.Equal(t, 404, resp.StatusCode)
})
t.Run("GET IPFS invalid CID returns 400 (Bad Request)", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/QmInvalid/pleaseDontAddMe")
assert.Equal(t, 400, resp.StatusCode)
})
t.Run("GET IPFS inlined zero-length data object returns ok code (200)", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/bafkqaaa")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length"))
assert.Equal(t, "", resp.Body)
})
t.Run("GET IPFS inlined zero-length data object with byte range returns ok code (200)", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/bafkqaaa", client.WithHeader("Range", "bytes=0-1048575"))
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "0", resp.Resp.Header.Get("Content-Length"))
assert.Equal(t, "text/plain", resp.Resp.Header.Get("Content-Type"))
})
t.Run("GET /ipfs/ipfs/{cid} returns redirect to the valid path", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/ipfs/bafkqaaa?query=to-remember")
assert.Contains(t,
resp.Body,
`<meta http-equiv="refresh" content="10;url=/ipfs/bafkqaaa?query=to-remember" />`,
)
assert.Contains(t,
resp.Body,
`<link rel="canonical" href="/ipfs/bafkqaaa?query=to-remember" />`,
)
})
})
t.Run("IPNS", func(t *testing.T) {
t.Parallel()
node.IPFS("name", "publish", "--allow-offline", cid)
t.Run("GET invalid IPNS root returns 400 (Bad Request)", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipns/QmInvalid/pleaseDontAddMe")
assert.Equal(t, 400, resp.StatusCode)
})
t.Run("GET IPNS path succeeds", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipns/{{.PeerID}}")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "Hello Worlds!", resp.Body)
})
t.Run("GET /ipfs/ipns/{peerid} returns redirect to the valid path", func(t *testing.T) {
t.Parallel()
resp := client.Get("/ipfs/ipns/{{.PeerID}}?query=to-remember")
peerID := node.PeerID().String()
assert.Contains(t,
resp.Body,
fmt.Sprintf(`<meta http-equiv="refresh" content="10;url=/ipns/%s?query=to-remember" />`, peerID),
)
assert.Contains(t,
resp.Body,
fmt.Sprintf(`<link rel="canonical" href="/ipns/%s?query=to-remember" />`, peerID),
)
})
})
t.Run("GET invalid IPFS path errors", func(t *testing.T) {
t.Parallel()
assert.Equal(t, 400, client.Get("/ipfs/12345").StatusCode)
})
t.Run("GET invalid path errors", func(t *testing.T) {
t.Parallel()
assert.Equal(t, 404, client.Get("/12345").StatusCode)
})
// TODO: these tests that use the API URL shouldn't be part of gateway tests...
t.Run("GET /webui returns 301 or 302", func(t *testing.T) {
t.Parallel()
resp := node.APIClient().DisableRedirects().Get("/webui")
assert.Contains(t, []int{302, 301}, resp.StatusCode)
})
t.Run("GET /webui/ returns 301 or 302", func(t *testing.T) {
t.Parallel()
resp := node.APIClient().DisableRedirects().Get("/webui/")
assert.Contains(t, []int{302, 301}, resp.StatusCode)
})
t.Run("GET /logs returns logs", func(t *testing.T) {
t.Parallel()
apiClient := node.APIClient()
reqURL := apiClient.BuildURL("/logs")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
require.NoError(t, err)
resp, err := apiClient.Client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
// read the first line of the output and parse its JSON
dec := json.NewDecoder(resp.Body)
event := struct{ Event string }{}
err = dec.Decode(&event)
require.NoError(t, err)
assert.Equal(t, "log API client connected", event.Event)
})
t.Run("POST /api/v0/version succeeds", func(t *testing.T) {
t.Parallel()
resp := node.APIClient().Post("/api/v0/version", nil)
assert.Equal(t, 200, resp.StatusCode)
assert.Len(t, resp.Resp.TransferEncoding, 1)
assert.Equal(t, "chunked", resp.Resp.TransferEncoding[0])
vers := struct{ Version string }{}
err := json.Unmarshal([]byte(resp.Body), &vers)
require.NoError(t, err)
assert.NotEmpty(t, vers.Version)
})
t.Run("pprof", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init().StartDaemon()
apiClient := node.APIClient()
t.Run("mutex", func(t *testing.T) {
t.Parallel()
t.Run("setting the mutex fraction works (negative so it doesn't enable)", func(t *testing.T) {
t.Parallel()
resp := apiClient.Post("/debug/pprof-mutex/?fraction=-1", nil)
assert.Equal(t, 200, resp.StatusCode)
})
t.Run("mutex endpoint doesn't accept a string as an argument", func(t *testing.T) {
t.Parallel()
resp := apiClient.Post("/debug/pprof-mutex/?fraction=that_is_a_string", nil)
assert.Equal(t, 400, resp.StatusCode)
})
t.Run("mutex endpoint returns 405 on GET", func(t *testing.T) {
t.Parallel()
resp := apiClient.Get("/debug/pprof-mutex/?fraction=-1")
assert.Equal(t, 405, resp.StatusCode)
})
})
t.Run("block", func(t *testing.T) {
t.Parallel()
t.Run("setting the block profiler rate works (0 so it doesn't enable)", func(t *testing.T) {
t.Parallel()
resp := apiClient.Post("/debug/pprof-block/?rate=0", nil)
assert.Equal(t, 200, resp.StatusCode)
})
t.Run("block profiler endpoint doesn't accept a string as an argument", func(t *testing.T) {
t.Parallel()
resp := apiClient.Post("/debug/pprof-block/?rate=that_is_a_string", nil)
assert.Equal(t, 400, resp.StatusCode)
})
t.Run("block profiler endpoint returns 405 on GET", func(t *testing.T) {
t.Parallel()
resp := apiClient.Get("/debug/pprof-block/?rate=0")
assert.Equal(t, 405, resp.StatusCode)
})
})
})
t.Run("index content types", func(t *testing.T) {
t.Parallel()
h := harness.NewT(t)
node := h.NewNode().Init().StartDaemon()
h.WriteFile("index/index.html", "<p></p>")
cid := node.IPFS("add", "-Q", "-r", filepath.Join(h.Dir, "index")).Stderr.Trimmed()
apiClient := node.APIClient()
apiClient.TemplateData = map[string]string{"CID": cid}
t.Run("GET index.html has correct content type", func(t *testing.T) {
t.Parallel()
res := apiClient.Get("/ipfs/{{.CID}}/")
assert.Equal(t, "text/html; charset=utf-8", res.Resp.Header.Get("Content-Type"))
})
t.Run("HEAD index.html has no content", func(t *testing.T) {
t.Parallel()
res := apiClient.Head("/ipfs/{{.CID}}/")
assert.Equal(t, "", res.Body)
assert.Equal(t, "", res.Resp.Header.Get("Content-Length"))
})
})
t.Run("readonly API", func(t *testing.T) {
t.Parallel()
client := node.GatewayClient()
fileContents := "12345"
h.WriteFile("readonly/dir/test", fileContents)
cids := node.IPFS("add", "-r", "-q", filepath.Join(h.Dir, "readonly/dir")).Stdout.Lines()
rootCID := cids[len(cids)-1]
client.TemplateData = map[string]string{"RootCID": rootCID}
t.Run("Get IPFS directory file through readonly API succeeds", func(t *testing.T) {
t.Parallel()
resp := client.Get("/api/v0/cat?arg={{.RootCID}}/test")
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, fileContents, resp.Body)
})
t.Run("refs IPFS directory file through readonly API succeeds", func(t *testing.T) {
t.Parallel()
resp := client.Get("/api/v0/refs?arg={{.RootCID}}/test")
assert.Equal(t, 200, resp.StatusCode)
})
t.Run("test gateway API is sanitized", func(t *testing.T) {
t.Parallel()
for _, cmd := range []string{
"add",
"block/put",
"bootstrap",
"config",
"dag/put",
"dag/import",
"dht",
"diag",
"id",
"mount",
"name/publish",
"object/put",
"object/new",
"object/patch",
"pin",
"ping",
"repo",
"stats",
"swarm",
"file",
"update",
"bitswap",
} {
t.Run(cmd, func(t *testing.T) {
cmd := cmd
t.Parallel()
assert.Equal(t, 404, client.Get("/api/v0/"+cmd).StatusCode)
})
}
})
})
t.Run("refs/local", func(t *testing.T) {
t.Parallel()
gatewayAddr := URLStrToMultiaddr(node.GatewayURL())
res := node.RunIPFS("--api", gatewayAddr.String(), "refs", "local")
assert.Equal(t,
`Error: invalid path "local": selected encoding not supported`,
res.Stderr.Trimmed(),
)
})
t.Run("raw leaves node", func(t *testing.T) {
t.Parallel()
contents := "This is RAW!"
cid := node.IPFSAddStr(contents, "--raw-leaves")
assert.Equal(t, contents, client.Get("/ipfs/"+cid).Body)
})
t.Run("compact blocks", func(t *testing.T) {
t.Parallel()
block1 := "\x0a\x09\x08\x02\x12\x03\x66\x6f\x6f\x18\x03"
block2 := "\x0a\x04\x08\x02\x18\x06\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c" +
"\x8b\x05\xeb\x66\x2b\xe0\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65" +
"\x12\x24\x0a\x22\x12\x20\xcf\x92\xfd\xef\xcd\xc3\x4c\xac\x00\x9c\x8b\x05\xeb\x66\x2b\xe0" +
"\x61\x8d\xb9\xde\x55\xec\xd4\x27\x85\xe9\xec\x67\x12\xf8\xdf\x65"
node.PipeStrToIPFS(block1, "block", "put")
block2CID := node.PipeStrToIPFS(block2, "block", "put", "--cid-codec=dag-pb").Stdout.Trimmed()
resp := client.Get("/ipfs/" + block2CID)
assert.Equal(t, 200, resp.StatusCode)
assert.Equal(t, "foofoo", resp.Body)
})
t.Run("verify gateway file", func(t *testing.T) {
t.Parallel()
r := regexp.MustCompile(`Gateway \(readonly\) server listening on (?P<addr>.+)\s`)
matches := r.FindStringSubmatch(node.Daemon.Stdout.String())
ma, err := multiaddr.NewMultiaddr(matches[1])
require.NoError(t, err)
netAddr, err := manet.ToNetAddr(ma)
require.NoError(t, err)
expURL := "http://" + netAddr.String()
b, err := os.ReadFile(filepath.Join(node.Dir, "gateway"))
require.NoError(t, err)
assert.Equal(t, expURL, string(b))
})
t.Run("verify gateway file diallable while on unspecified", func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Addresses.Gateway = config.Strings{"/ip4/127.0.0.1/tcp/32563"}
})
node.StartDaemon()
b, err := os.ReadFile(filepath.Join(node.Dir, "gateway"))
require.NoError(t, err)
assert.Equal(t, "http://127.0.0.1:32563", string(b))
})
t.Run("NoFetch", func(t *testing.T) {
t.Parallel()
nodes := harness.NewT(t).NewNodes(2).Init()
node1 := nodes[0]
node2 := nodes[1]
node1.UpdateConfig(func(cfg *config.Config) {
cfg.Gateway.NoFetch = true
})
nodes.StartDaemons().Connect()
t.Run("not present", func(t *testing.T) {
cidFoo := node2.IPFSAddStr("foo")
t.Run("not present key from node 1", func(t *testing.T) {
t.Parallel()
assert.Equal(t, 404, node1.GatewayClient().Get("/ipfs/"+cidFoo).StatusCode)
})
t.Run("not present IPNS key from node 1", func(t *testing.T) {
t.Parallel()
assert.Equal(t, 400, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode)
})
})
t.Run("present", func(t *testing.T) {
cidBar := node1.IPFSAddStr("bar")
t.Run("present key from node 1", func(t *testing.T) {
t.Parallel()
assert.Equal(t, 200, node1.GatewayClient().Get("/ipfs/"+cidBar).StatusCode)
})
t.Run("present IPNS key from node 1", func(t *testing.T) {
t.Parallel()
node2.IPFS("name", "publish", "/ipfs/"+cidBar)
assert.Equal(t, 200, node1.GatewayClient().Get("/ipns/"+node2.PeerID().String()).StatusCode)
})
})
})
}

View File

@ -119,15 +119,19 @@ func (h *Harness) TempFile() *os.File {
}
// WriteFile writes a file given a filename and its contents.
// The filename should be a relative path.
// The filename must be a relative path, or this panics.
func (h *Harness) WriteFile(filename, contents string) {
if filepath.IsAbs(filename) {
log.Panicf("%s must be a relative path", filename)
}
absPath := filepath.Join(h.Runner.Dir, filename)
err := os.WriteFile(absPath, []byte(contents), 0644)
err := os.MkdirAll(filepath.Dir(absPath), 0777)
if err != nil {
log.Panicf("writing '%s' ('%s'): %s", filename, absPath, err.Error())
log.Panicf("creating intermediate dirs for %q: %s", filename, err.Error())
}
err = os.WriteFile(absPath, []byte(contents), 0644)
if err != nil {
log.Panicf("writing %q (%q): %s", filename, absPath, err.Error())
}
}
@ -140,8 +144,7 @@ func WaitForFile(path string, timeout time.Duration) error {
for {
select {
case <-timer.C:
end := time.Now()
return fmt.Errorf("timeout waiting for %s after %v", path, end.Sub(start))
return fmt.Errorf("timeout waiting for %s after %v", path, time.Since(start))
case <-ticker.C:
_, err := os.Stat(path)
if err == nil {

View File

@ -0,0 +1,116 @@
package harness
import (
"io"
"net/http"
"strings"
"text/template"
"time"
)
// HTTPClient is an HTTP client with some conveniences for testing.
// URLs are constructed from a base URL.
// The response body is buffered into a string.
// Internal errors cause panics so that tests don't need to check errors.
// The paths are evaluated as Go templates for readable string interpolation.
type HTTPClient struct {
Client *http.Client
BaseURL string
Timeout time.Duration
TemplateData any
}
type HTTPResponse struct {
Body string
StatusCode int
Headers http.Header
// The raw response. The body will be closed on this response.
Resp *http.Response
}
func (c *HTTPClient) WithHeader(k, v string) func(h *http.Request) {
return func(h *http.Request) {
h.Header.Add(k, v)
}
}
func (c *HTTPClient) DisableRedirects() *HTTPClient {
c.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
return c
}
// Do executes the request unchanged.
func (c *HTTPClient) Do(req *http.Request) *HTTPResponse {
log.Debugf("making HTTP req %s to %q with headers %+v", req.Method, req.URL.String(), req.Header)
resp, err := c.Client.Do(req)
if resp != nil && resp.Body != nil {
defer resp.Body.Close()
}
if err != nil {
panic(err)
}
bodyStr, err := io.ReadAll(resp.Body)
if err != nil {
panic(err)
}
return &HTTPResponse{
Body: string(bodyStr),
StatusCode: resp.StatusCode,
Headers: resp.Header,
Resp: resp,
}
}
// BuildURL constructs a request URL from the given path by interpolating the string and then appending it to the base URL.
func (c *HTTPClient) BuildURL(urlPath string) string {
sb := &strings.Builder{}
err := template.Must(template.New("test").Parse(urlPath)).Execute(sb, c.TemplateData)
if err != nil {
panic(err)
}
renderedPath := sb.String()
return c.BaseURL + renderedPath
}
func (c *HTTPClient) Get(urlPath string, opts ...func(*http.Request)) *HTTPResponse {
req, err := http.NewRequest(http.MethodGet, c.BuildURL(urlPath), nil)
if err != nil {
panic(err)
}
for _, o := range opts {
o(req)
}
return c.Do(req)
}
func (c *HTTPClient) Post(urlPath string, body io.Reader, opts ...func(*http.Request)) *HTTPResponse {
req, err := http.NewRequest(http.MethodPost, c.BuildURL(urlPath), body)
if err != nil {
panic(err)
}
for _, o := range opts {
o(req)
}
return c.Do(req)
}
func (c *HTTPClient) PostStr(urlpath, body string, opts ...func(*http.Request)) *HTTPResponse {
r := strings.NewReader(body)
return c.Post(urlpath, r, opts...)
}
func (c *HTTPClient) Head(urlPath string, opts ...func(*http.Request)) *HTTPResponse {
req, err := http.NewRequest(http.MethodHead, c.BuildURL(urlPath), nil)
if err != nil {
panic(err)
}
for _, o := range opts {
o(req)
}
return c.Do(req)
}

View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"os/exec"
@ -19,6 +20,7 @@ import (
serial "github.com/ipfs/kubo/config/serialize"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr/net"
)
var log = logging.Logger("testharness")
@ -29,14 +31,15 @@ type Node struct {
ID int
Dir string
APIListenAddr multiaddr.Multiaddr
SwarmAddr multiaddr.Multiaddr
EnableMDNS bool
APIListenAddr multiaddr.Multiaddr
GatewayListenAddr multiaddr.Multiaddr
SwarmAddr multiaddr.Multiaddr
EnableMDNS bool
IPFSBin string
Runner *Runner
daemon *RunResult
Daemon *RunResult
}
func BuildNode(ipfsBin, baseDir string, id int) *Node {
@ -134,11 +137,19 @@ func (n *Node) Init(ipfsArgs ...string) *Node {
n.APIListenAddr = apiAddr
}
if n.GatewayListenAddr == nil {
gatewayAddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
if err != nil {
panic(err)
}
n.GatewayListenAddr = gatewayAddr
}
n.UpdateConfig(func(cfg *config.Config) {
cfg.Bootstrap = []string{}
cfg.Addresses.Swarm = []string{n.SwarmAddr.String()}
cfg.Addresses.API = []string{n.APIListenAddr.String()}
cfg.Addresses.Gateway = []string{""}
cfg.Addresses.Gateway = []string{n.GatewayListenAddr.String()}
cfg.Swarm.DisableNatPortMap = true
cfg.Discovery.MDNS.Enabled = n.EnableMDNS
})
@ -159,7 +170,7 @@ func (n *Node) StartDaemon(ipfsArgs ...string) *Node {
RunFunc: (*exec.Cmd).Start,
})
n.daemon = &res
n.Daemon = &res
log.Debugf("node %d started, checking API", n.ID)
n.WaitOnAPI()
@ -167,7 +178,7 @@ func (n *Node) StartDaemon(ipfsArgs ...string) *Node {
}
func (n *Node) signalAndWait(watch <-chan struct{}, signal os.Signal, t time.Duration) bool {
err := n.daemon.Cmd.Process.Signal(signal)
err := n.Daemon.Cmd.Process.Signal(signal)
if err != nil {
if errors.Is(err, os.ErrProcessDone) {
log.Debugf("process for node %d has already finished", n.ID)
@ -187,13 +198,13 @@ func (n *Node) signalAndWait(watch <-chan struct{}, signal os.Signal, t time.Dur
func (n *Node) StopDaemon() *Node {
log.Debugf("stopping node %d", n.ID)
if n.daemon == nil {
if n.Daemon == nil {
log.Debugf("didn't stop node %d since no daemon present", n.ID)
return n
}
watch := make(chan struct{}, 1)
go func() {
_, _ = n.daemon.Cmd.Process.Wait()
_, _ = n.Daemon.Cmd.Process.Wait()
watch <- struct{}{}
}()
log.Debugf("signaling node %d with SIGTERM", n.ID)
@ -224,6 +235,15 @@ func (n *Node) APIAddr() multiaddr.Multiaddr {
return ma
}
func (n *Node) APIURL() string {
apiAddr := n.APIAddr()
netAddr, err := manet.ToNetAddr(apiAddr)
if err != nil {
panic(err)
}
return "http://" + netAddr.String()
}
func (n *Node) TryAPIAddr() (multiaddr.Multiaddr, error) {
b, err := os.ReadFile(filepath.Join(n.Dir, "api"))
if err != nil {
@ -305,20 +325,21 @@ func (n *Node) WaitOnAPI() *Node {
log.Debugf("waiting on API for node %d", n.ID)
for i := 0; i < 50; i++ {
if n.checkAPI() {
log.Debugf("daemon API found, daemon stdout: %s", n.Daemon.Stdout.String())
return n
}
time.Sleep(400 * time.Millisecond)
}
log.Panicf("node %d with peer ID %s failed to come online: \n%s\n\n%s", n.ID, n.PeerID(), n.daemon.Stderr.String(), n.daemon.Stdout.String())
log.Panicf("node %d with peer ID %s failed to come online: \n%s\n\n%s", n.ID, n.PeerID(), n.Daemon.Stderr.String(), n.Daemon.Stdout.String())
return n
}
func (n *Node) IsAlive() bool {
if n.daemon == nil || n.daemon.Cmd == nil || n.daemon.Cmd.Process == nil {
if n.Daemon == nil || n.Daemon.Cmd == nil || n.Daemon.Cmd.Process == nil {
return false
}
log.Debugf("signaling node %d daemon process for liveness check", n.ID)
err := n.daemon.Cmd.Process.Signal(syscall.Signal(0))
err := n.Daemon.Cmd.Process.Signal(syscall.Signal(0))
if err == nil {
log.Debugf("node %d daemon is alive", n.ID)
return true
@ -381,3 +402,38 @@ func (n *Node) Peers() []multiaddr.Multiaddr {
}
return addrs
}
// GatewayURL waits for the gateway file and then returns its contents or times out.
func (n *Node) GatewayURL() string {
timer := time.NewTimer(1 * time.Second)
defer timer.Stop()
for {
select {
case <-timer.C:
panic("timeout waiting for gateway file")
default:
b, err := os.ReadFile(filepath.Join(n.Dir, "gateway"))
if err == nil {
return strings.TrimSpace(string(b))
}
if !errors.Is(err, fs.ErrNotExist) {
panic(err)
}
time.Sleep(1 * time.Millisecond)
}
}
}
func (n *Node) GatewayClient() *HTTPClient {
return &HTTPClient{
Client: http.DefaultClient,
BaseURL: n.GatewayURL(),
}
}
func (n *Node) APIClient() *HTTPClient {
return &HTTPClient{
Client: http.DefaultClient,
BaseURL: n.APIURL(),
}
}

View File

@ -1,6 +1,8 @@
package harness
import (
"sync"
"github.com/multiformats/go-multiaddr"
)
@ -15,14 +17,22 @@ func (n Nodes) Init(args ...string) Nodes {
}
func (n Nodes) Connect() Nodes {
wg := sync.WaitGroup{}
for i, node := range n {
for j, otherNode := range n {
if i == j {
continue
}
node.Connect(otherNode)
node := node
otherNode := otherNode
wg.Add(1)
go func() {
defer wg.Done()
node.Connect(otherNode)
}()
}
}
wg.Wait()
for _, node := range n {
firstPeer := node.Peers()[0]
if _, err := firstPeer.ValueForProtocol(multiaddr.P_P2P); err != nil {
@ -33,9 +43,16 @@ func (n Nodes) Connect() Nodes {
}
func (n Nodes) StartDaemons() Nodes {
wg := sync.WaitGroup{}
for _, node := range n {
node.StartDaemon()
wg.Add(1)
node := node
go func() {
defer wg.Done()
node.StartDaemon()
}()
}
wg.Wait()
return n
}

View File

@ -3,7 +3,13 @@ package testutils
import (
"bufio"
"fmt"
"net"
"net/netip"
"net/url"
"strings"
"github.com/multiformats/go-multiaddr"
manet "github.com/multiformats/go-multiaddr/net"
)
// StrCat takes a bunch of strings or string slices
@ -51,3 +57,21 @@ func SplitLines(s string) []string {
}
return lines
}
// URLStrToMultiaddr converts a URL string like http://localhost:80 to a multiaddr.
func URLStrToMultiaddr(u string) multiaddr.Multiaddr {
parsedURL, err := url.Parse(u)
if err != nil {
panic(err)
}
addrPort, err := netip.ParseAddrPort(parsedURL.Host)
if err != nil {
panic(err)
}
tcpAddr := net.TCPAddrFromAddrPort(addrPort)
ma, err := manet.FromNetAddr(tcpAddr)
if err != nil {
panic(err)
}
return ma
}

View File

@ -1,2 +0,0 @@
foo

View File

@ -1,357 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2015 Matt Bell
# MIT Licensed; see the LICENSE file in this repository.
#
test_description="Test HTTP Gateway"
. lib/test-lib.sh
test_init_ipfs
test_launch_ipfs_daemon
port=$GWAY_PORT
apiport=$API_PORT
# TODO check both 5001 and 5002.
# 5001 should have a readable gateway (part of the API)
# 5002 should have a readable gateway (using ipfs config Addresses.Gateway)
# but ideally we should only write the tests once. so maybe we need to
# define a function to test a gateway, and do so for each port.
# for now we check 5001 here as 5002 will be checked in gateway-writable.
test_expect_success "Make a file to test with" '
echo "Hello Worlds!" >expected &&
HASH=$(ipfs add -q expected) ||
test_fsh cat daemon_err
'
test_expect_success "GET IPFS path succeeds" '
curl -sfo actual "http://127.0.0.1:$port/ipfs/$HASH"
'
test_expect_success "GET IPFS path with explicit ?filename succeeds with proper header" "
curl -fo actual -D actual_headers 'http://127.0.0.1:$port/ipfs/$HASH?filename=testтест.pdf' &&
grep -F 'Content-Disposition: inline; filename=\"test____.pdf\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.pdf' actual_headers
"
test_expect_success "GET IPFS path with explicit ?filename and &download=true succeeds with proper header" "
curl -fo actual -D actual_headers 'http://127.0.0.1:$port/ipfs/$HASH?filename=testтест.mp4&download=true' &&
grep -F 'Content-Disposition: attachment; filename=\"test____.mp4\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.mp4' actual_headers
"
# https://github.com/ipfs/go-ipfs/issues/4025#issuecomment-342250616
test_expect_success "GET for Service Worker registration outside of an IPFS content root errors" "
curl -H 'Service-Worker: script' -svX GET 'http://127.0.0.1:$port/ipfs/$HASH?filename=sw.js' > curl_sw_out 2>&1 &&
grep 'HTTP/1.1 400 Bad Request' curl_sw_out &&
grep 'navigator.serviceWorker: registration is not allowed for this scope' curl_sw_out
"
test_expect_success "GET IPFS path output looks good" '
test_cmp expected actual &&
rm actual
'
test_expect_success "GET IPFS directory path succeeds" '
mkdir -p dir/dirwithindex &&
echo "12345" >dir/test &&
echo "hello i am a webpage" >dir/dirwithindex/index.html &&
ipfs add -r -q dir >actual &&
HASH2=$(tail -n 1 actual) &&
curl -sf "http://127.0.0.1:$port/ipfs/$HASH2"
'
test_expect_success "GET IPFS directory file succeeds" '
curl -sfo actual "http://127.0.0.1:$port/ipfs/$HASH2/test"
'
test_expect_success "GET IPFS directory file output looks good" '
test_cmp dir/test actual
'
test_expect_success "GET IPFS directory with index.html returns redirect to add trailing slash" "
curl -sI -o response_without_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex?query=to-remember\" &&
test_should_contain \"HTTP/1.1 301 Moved Permanently\" response_without_slash &&
test_should_contain \"Location: /ipfs/$HASH2/dirwithindex/?query=to-remember\" response_without_slash
"
# This enables go get to parse go-import meta tags from index.html files stored in IPFS
# https://github.com/ipfs/kubo/pull/3963
test_expect_success "GET IPFS directory with index.html and no trailing slash returns expected output when go-get is passed" "
curl -s -o response_with_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex?go-get=1\" &&
test_should_contain \"hello i am a webpage\" response_with_slash
"
test_expect_success "GET IPFS directory with index.html and trailing slash returns expected output" "
curl -s -o response_with_slash \"http://127.0.0.1:$port/ipfs/$HASH2/dirwithindex/?query=to-remember\" &&
test_should_contain \"hello i am a webpage\" response_with_slash
"
test_expect_success "GET IPFS nonexistent file returns 404 (Not Found)" '
test_curl_resp_http_code "http://127.0.0.1:$port/ipfs/$HASH2/pleaseDontAddMe" "HTTP/1.1 404 Not Found"
'
test_expect_success "GET IPFS invalid CID returns 400 (Bad Request)" '
test_curl_resp_http_code "http://127.0.0.1:$port/ipfs/QmInvalid/pleaseDontAddMe" "HTTP/1.1 400 Bad Request"
'
# https://github.com/ipfs/go-ipfs/issues/8230
test_expect_success "GET IPFS inlined zero-length data object returns ok code (200)" '
curl -sD - "http://127.0.0.1:$port/ipfs/bafkqaaa" > empty_ok_response &&
test_should_contain "HTTP/1.1 200 OK" empty_ok_response &&
test_should_contain "Content-Length: 0" empty_ok_response
'
# https://github.com/ipfs/kubo/issues/9238
test_expect_success "GET IPFS inlined zero-length data object with byte range returns ok code (200)" '
curl -sD - "http://127.0.0.1:$port/ipfs/bafkqaaa" -H "Range: bytes=0-1048575" > empty_ok_response &&
test_should_contain "HTTP/1.1 200 OK" empty_ok_response &&
test_should_contain "Content-Length: 0" empty_ok_response &&
test_should_contain "Content-Type: text/plain" empty_ok_response
'
test_expect_success "GET /ipfs/ipfs/{cid} returns redirect to the valid path" '
curl -sD - "http://127.0.0.1:$port/ipfs/ipfs/bafkqaaa?query=to-remember" > response_with_double_ipfs_ns &&
test_should_contain "<meta http-equiv=\"refresh\" content=\"10;url=/ipfs/bafkqaaa?query=to-remember\" />" response_with_double_ipfs_ns &&
test_should_contain "<link rel=\"canonical\" href=\"/ipfs/bafkqaaa?query=to-remember\" />" response_with_double_ipfs_ns
'
test_expect_success "GET invalid IPNS root returns 400 (Bad Request)" '
test_curl_resp_http_code "http://127.0.0.1:$port/ipns/QmInvalid/pleaseDontAddMe" "HTTP/1.1 400 Bad Request"
'
test_expect_success "GET IPNS path succeeds" '
ipfs name publish --allow-offline "$HASH" &&
PEERID=$(ipfs config Identity.PeerID) &&
test_check_peerid "$PEERID" &&
curl -sfo actual "http://127.0.0.1:$port/ipns/$PEERID"
'
test_expect_success "GET IPNS path output looks good" '
test_cmp expected actual
'
test_expect_success "GET /ipfs/ipns/{peerid} returns redirect to the valid path" '
PEERID=$(ipfs config Identity.PeerID) &&
curl -sD - "http://127.0.0.1:$port/ipfs/ipns/${PEERID}?query=to-remember" > response_with_ipfs_ipns_ns &&
test_should_contain "<meta http-equiv=\"refresh\" content=\"10;url=/ipns/${PEERID}?query=to-remember\" />" response_with_ipfs_ipns_ns &&
test_should_contain "<link rel=\"canonical\" href=\"/ipns/${PEERID}?query=to-remember\" />" response_with_ipfs_ipns_ns
'
test_expect_success "GET invalid IPFS path errors" '
test_must_fail curl -sf "http://127.0.0.1:$port/ipfs/12345"
'
test_expect_success "GET invalid path errors" '
test_must_fail curl -sf "http://127.0.0.1:$port/12345"
'
test_expect_success "GET /webui returns code expected" '
test_curl_resp_http_code "http://127.0.0.1:$apiport/webui" "HTTP/1.1 302 Found" "HTTP/1.1 301 Moved Permanently"
'
test_expect_success "GET /webui/ returns code expected" '
test_curl_resp_http_code "http://127.0.0.1:$apiport/webui/" "HTTP/1.1 302 Found" "HTTP/1.1 301 Moved Permanently"
'
test_expect_success "GET /logs returns logs" '
test_expect_code 28 curl http://127.0.0.1:$apiport/logs -m1 > log_out
'
test_expect_success "log output looks good" '
grep "log API client connected" log_out
'
test_expect_success "GET /api/v0/version succeeds" '
curl -X POST -v "http://127.0.0.1:$apiport/api/v0/version" 2> version_out
'
test_expect_success "output only has one transfer encoding header" '
grep "Transfer-Encoding: chunked" version_out | wc -l | xargs echo > tecount_out &&
echo "1" > tecount_exp &&
test_cmp tecount_out tecount_exp
'
curl_pprofmutex() {
curl -f -X POST "http://127.0.0.1:$apiport/debug/pprof-mutex/?fraction=$1"
}
test_expect_success "set mutex fraction for pprof (negative so it doesn't enable)" '
curl_pprofmutex -1
'
test_expect_success "test failure conditions of mutex pprof endpoint" '
test_must_fail curl_pprofmutex &&
test_must_fail curl_pprofmutex that_is_string &&
test_must_fail curl -f -X GET "http://127.0.0.1:$apiport/debug/pprof-mutex/?fraction=-1"
'
curl_pprofblock() {
curl -f -X POST "http://127.0.0.1:$apiport/debug/pprof-block/?rate=$1"
}
test_expect_success "set blocking profiler rate for pprof (0 so it doesn't enable)" '
curl_pprofblock 0
'
test_expect_success "test failure conditions of mutex block endpoint" '
test_must_fail curl_pprofblock &&
test_must_fail curl_pprofblock that_is_string &&
test_must_fail curl -f -X GET "http://127.0.0.1:$apiport/debug/pprof-block/?rate=0"
'
test_expect_success "setup index hash" '
mkdir index &&
echo "<p></p>" > index/index.html &&
INDEXHASH=$(ipfs add -Q -r index)
echo index: $INDEXHASH
'
test_expect_success "GET 'index.html' has correct content type" '
curl -I "http://127.0.0.1:$port/ipfs/$INDEXHASH/" > indexout
'
test_expect_success "output looks good" '
grep "Content-Type: text/html" indexout
'
test_expect_success "HEAD 'index.html' has no content" '
curl -X HEAD --max-time 1 http://127.0.0.1:$port/ipfs/$INDEXHASH/ > output;
[ ! -s output ]
'
# test ipfs readonly api
test_curl_gateway_api() {
curl -sfo actual "http://127.0.0.1:$port/api/v0/$1"
}
test_expect_success "get IPFS directory file through readonly API succeeds" '
test_curl_gateway_api "cat?arg=$HASH2/test"
'
test_expect_success "get IPFS directory file through readonly API output looks good" '
test_cmp dir/test actual
'
test_expect_success "refs IPFS directory file through readonly API succeeds" '
test_curl_gateway_api "refs?arg=$HASH2/test"
'
for cmd in add \
block/put \
bootstrap \
config \
dag/put \
dag/import \
dht \
diag \
id \
mount \
name/publish \
object/put \
object/new \
object/patch \
pin \
ping \
repo \
stats \
swarm \
file \
update \
bitswap
do
test_expect_success "test gateway api is sanitized: $cmd" '
test_curl_resp_http_code "http://127.0.0.1:$port/api/v0/$cmd" "HTTP/1.1 404 Not Found"
'
done
# This one is different. `local` will be interpreted as a path if the command isn't defined.
test_expect_success "test gateway api is sanitized: refs/local" '
echo "Error: invalid path \"local\": selected encoding not supported" > refs_local_expected &&
! ipfs --api /ip4/127.0.0.1/tcp/$port refs local > refs_local_actual 2>&1 &&
test_cmp refs_local_expected refs_local_actual
'
test_expect_success "create raw-leaves node" '
echo "This is RAW!" > rfile &&
echo "This is RAW!" | ipfs add --raw-leaves -q > rhash
'
test_expect_success "try fetching it from gateway" '
curl http://127.0.0.1:$port/ipfs/$(cat rhash) > ffile &&
test_cmp rfile ffile
'
test_expect_success "Add compact blocks" '
ipfs block put ../t0110-gateway-data/foo.block &&
FOO2_HASH=$(ipfs block put --cid-codec=dag-pb ../t0110-gateway-data/foofoo.block) &&
printf "foofoo" > expected
'
test_expect_success "GET compact blocks succeeds" '
curl -o actual "http://127.0.0.1:$port/ipfs/$FOO2_HASH" &&
test_cmp expected actual
'
test_expect_success "Verify gateway file" '
cat "$IPFS_PATH/gateway" > gateway_file_actual &&
echo -n "http://$GWAY_ADDR" > gateway_daemon_actual &&
test_cmp gateway_daemon_actual gateway_file_actual
'
test_kill_ipfs_daemon
GWPORT=32563
test_expect_success "Verify gateway file diallable while on unspecified" '
ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/$GWPORT &&
test_launch_ipfs_daemon &&
cat "$IPFS_PATH/gateway" > gateway_file_actual &&
echo -n "http://127.0.0.1:$GWPORT" > gateway_file_expected &&
test_cmp gateway_file_expected gateway_file_actual
'
test_kill_ipfs_daemon
test_expect_success "set up iptb testbed" '
iptb testbed create -type localipfs -count 5 -force -init &&
ipfsi 0 config Addresses.Gateway /ip4/127.0.0.1/tcp/$GWPORT &&
PEERID_1=$(iptb attr get 1 id)
'
test_expect_success "set NoFetch to true in config of node 0" '
ipfsi 0 config --bool=true Gateway.NoFetch true
'
test_expect_success "start ipfs nodes" '
iptb start -wait &&
iptb connect 0 1
'
test_expect_success "try fetching not present key from node 0" '
FOO=$(echo "foo" | ipfsi 1 add -Q) &&
test_expect_code 22 curl -f "http://127.0.0.1:$GWPORT/ipfs/$FOO"
'
test_expect_success "try fetching not present ipns key from node 0" '
ipfsi 1 name publish /ipfs/$FOO &&
test_expect_code 22 curl -f "http://127.0.0.1:$GWPORT/ipns/$PEERID_1"
'
test_expect_success "try fetching present key from node 0" '
BAR=$(echo "bar" | ipfsi 0 add -Q) &&
curl -f "http://127.0.0.1:$GWPORT/ipfs/$BAR"
'
test_expect_success "try fetching present ipns key from node 0" '
ipfsi 1 name publish /ipfs/$BAR &&
curl "http://127.0.0.1:$GWPORT/ipns/$PEERID_1"
'
test_expect_success "stop testbed" '
iptb stop
'
test_done