mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
test: port remote pinning tests to Go (#9720)
This also means that rb-pinning-service-api is no longer required for running remote pinning tests. This alone saves at least 3 minutes in test runtime in CI because we don't need to checkout the repo, build the Docker image, run it, etc. Instead this implements a simple pinning service in Go that the test runs in-process, with a callback that can be used to control the async behavior of the pinning service (e.g. simulate work happening asynchronously like transitioning from "queued" -> "pinning" -> "pinned"). This also adds an environment variable to Kubo to control the MFS remote pin polling interval, so that we don't have to wait 30 seconds in the test for MFS changes to be repinned. This is purely for tests so I don't think we should document this. This entire test suite runs in around 2.5 sec on my laptop, compared to the existing 3+ minutes in CI.
This commit is contained in:
Gus Eggert
GitHub
parent
e89cce63fd
commit
a24cfb89a5
14
.github/workflows/sharness.yml
vendored
14
.github/workflows/sharness.yml
vendored
@ -29,24 +29,10 @@ jobs:
|
||||
path: kubo
|
||||
- name: Install missing tools
|
||||
run: sudo apt install -y socat net-tools fish libxml2-utils
|
||||
- name: Checkout IPFS Pinning Service API
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: ipfs-shipyard/rb-pinning-service-api
|
||||
ref: 773c3adbb421c551d2d89288abac3e01e1f7c3a8
|
||||
path: rb-pinning-service-api
|
||||
# TODO: check if docker compose (not docker-compose) is available on default gh runners
|
||||
- name: Start IPFS Pinning Service API
|
||||
run: |
|
||||
(for i in {1..3}; do docker compose pull && break || sleep 5; done) &&
|
||||
docker compose up -d
|
||||
working-directory: rb-pinning-service-api
|
||||
- name: Restore Go Cache
|
||||
uses: protocol/cache-go-action@v1
|
||||
with:
|
||||
name: ${{ github.job }}
|
||||
- name: Find IPFS Pinning Service API address
|
||||
run: echo "TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $GITHUB_ENV
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: test/sharness/lib/dependencies
|
||||
|
||||
@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
@ -31,7 +32,19 @@ func (x lastPin) IsValid() bool {
|
||||
return x != lastPin{}
|
||||
}
|
||||
|
||||
const daemonConfigPollInterval = time.Minute / 2
|
||||
var daemonConfigPollInterval = time.Minute / 2
|
||||
|
||||
func init() {
|
||||
// this environment variable is solely for testing, use at your own risk
|
||||
if pollDurStr := os.Getenv("MFS_PIN_POLL_INTERVAL"); pollDurStr != "" {
|
||||
d, err := time.ParseDuration(pollDurStr)
|
||||
if err != nil {
|
||||
mfslog.Error("error parsing MFS_PIN_POLL_INTERVAL, using default:", err)
|
||||
}
|
||||
daemonConfigPollInterval = d
|
||||
}
|
||||
}
|
||||
|
||||
const defaultRepinInterval = 5 * time.Minute
|
||||
|
||||
type pinMFSContext interface {
|
||||
|
||||
5
go.mod
5
go.mod
@ -43,6 +43,7 @@ require (
|
||||
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
|
||||
github.com/jbenet/go-temp-err-catcher v0.1.0
|
||||
github.com/jbenet/goprocess v0.1.4
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/libp2p/go-doh-resolver v0.4.0
|
||||
github.com/libp2p/go-libp2p v0.26.4
|
||||
github.com/libp2p/go-libp2p-http v0.4.0
|
||||
@ -67,6 +68,8 @@ require (
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/stretchr/testify v1.8.2
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
|
||||
github.com/tidwall/gjson v1.14.4
|
||||
github.com/tidwall/sjson v1.2.5
|
||||
github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1
|
||||
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
|
||||
go.opencensus.io v0.24.0
|
||||
@ -198,6 +201,8 @@ require (
|
||||
github.com/samber/lo v1.36.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
|
||||
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@ -495,6 +495,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
@ -858,6 +859,15 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb
|
||||
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
|
||||
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
|
||||
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
|
||||
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
|
||||
|
||||
@ -76,6 +76,23 @@ func (n *Node) WriteBytes(filename string, b []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// ReadFile reads the specific file. If it is relative, it is relative the node's root dir.
|
||||
func (n *Node) ReadFile(filename string) string {
|
||||
f := filename
|
||||
if !filepath.IsAbs(filename) {
|
||||
f = filepath.Join(n.Dir, filename)
|
||||
}
|
||||
b, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (n *Node) ConfigFile() string {
|
||||
return filepath.Join(n.Dir, "config")
|
||||
}
|
||||
|
||||
func (n *Node) ReadConfig() *config.Config {
|
||||
cfg, err := serial.Load(filepath.Join(n.Dir, "config"))
|
||||
if err != nil {
|
||||
|
||||
8
test/cli/must.go
Normal file
8
test/cli/must.go
Normal file
@ -0,0 +1,8 @@
|
||||
package cli
|
||||
|
||||
func MustVal[V any](val V, err error) V {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return val
|
||||
}
|
||||
446
test/cli/pinning_remote_test.go
Normal file
446
test/cli/pinning_remote_test.go
Normal file
@ -0,0 +1,446 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/ipfs/kubo/test/cli/testutils"
|
||||
"github.com/ipfs/kubo/test/cli/testutils/pinningservice"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
)
|
||||
|
||||
func runPinningService(t *testing.T, authToken string) (*pinningservice.PinningService, string) {
|
||||
svc := pinningservice.New()
|
||||
router := pinningservice.NewRouter(authToken, svc)
|
||||
server := &http.Server{Handler: router}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
go func() {
|
||||
err := server.Serve(listener)
|
||||
if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) {
|
||||
t.Logf("Serve error: %s", err)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() { listener.Close() })
|
||||
|
||||
return svc, fmt.Sprintf("http://%s/api/v1", listener.Addr().String())
|
||||
}
|
||||
|
||||
func TestRemotePinning(t *testing.T) {
|
||||
t.Parallel()
|
||||
authToken := "testauthtoken"
|
||||
|
||||
t.Run("MFS pinning", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.Runner.Env["MFS_PIN_POLL_INTERVAL"] = "10ms"
|
||||
|
||||
_, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.RepinInterval", `"1s"`)
|
||||
node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.PinName", `"test_pin"`)
|
||||
node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.Enable", "true")
|
||||
|
||||
node.StartDaemon()
|
||||
|
||||
node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-test-"+uuid.NewString())
|
||||
node.IPFS("files", "flush")
|
||||
res := node.IPFS("files", "stat", "/", "--enc=json")
|
||||
hash := gjson.Get(res.Stdout.String(), "Hash").Str
|
||||
|
||||
assert.Eventually(t,
|
||||
func() bool {
|
||||
res = node.IPFS("pin", "remote", "ls",
|
||||
"--service=svc",
|
||||
"--name=test_pin",
|
||||
"--status=queued,pinning,pinned,failed",
|
||||
"--enc=json",
|
||||
)
|
||||
pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
|
||||
return hash == pinnedHash
|
||||
},
|
||||
10*time.Second,
|
||||
10*time.Millisecond,
|
||||
)
|
||||
|
||||
t.Run("MFS root is repinned on CID change", func(t *testing.T) {
|
||||
node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-repin-test-"+uuid.NewString())
|
||||
node.IPFS("files", "flush")
|
||||
res = node.IPFS("files", "stat", "/", "--enc=json")
|
||||
hash := gjson.Get(res.Stdout.String(), "Hash").Str
|
||||
assert.Eventually(t,
|
||||
func() bool {
|
||||
res := node.IPFS("pin", "remote", "ls",
|
||||
"--service=svc",
|
||||
"--name=test_pin",
|
||||
"--status=queued,pinning,pinned,failed",
|
||||
"--enc=json",
|
||||
)
|
||||
pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
|
||||
return hash == pinnedHash
|
||||
},
|
||||
10*time.Second,
|
||||
10*time.Millisecond,
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// Pinning.RemoteServices includes API.Key, so we give it the same treatment
|
||||
// as Identity,PrivKey to prevent exposing it on the network
|
||||
t.Run("access token security", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
node.IPFS("pin", "remote", "service", "add", "1", "http://example1.com", "testkey")
|
||||
res := node.RunIPFS("config", "Pinning")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
|
||||
assert.NotContains(t, res.Stdout.String(), "testkey")
|
||||
|
||||
res = node.RunIPFS("config", "Pinning.RemoteServices.1.API.Key")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
|
||||
assert.NotContains(t, res.Stdout.String(), "testkey")
|
||||
|
||||
configShow := node.RunIPFS("config", "show").Stdout.String()
|
||||
assert.NotContains(t, configShow, "testkey")
|
||||
|
||||
t.Run("re-injecting config with 'ipfs config replace' preserves the API keys", func(t *testing.T) {
|
||||
node.WriteBytes("config-show", []byte(configShow))
|
||||
node.IPFS("config", "replace", "config-show")
|
||||
assert.Contains(t, node.ReadFile(node.ConfigFile()), "testkey")
|
||||
})
|
||||
|
||||
t.Run("injecting config with 'ipfs config replace' with API keys returns an error", func(t *testing.T) {
|
||||
// remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
|
||||
configJSON := MustVal(sjson.Delete(configShow, "Identity.PrivKey"))
|
||||
configJSON = MustVal(sjson.Set(configJSON, "Pinning.RemoteServices.1.API.Key", "testkey"))
|
||||
node.WriteBytes("new-config", []byte(configJSON))
|
||||
res := node.RunIPFS("config", "replace", "new-config")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "cannot change remote pinning services api info with `config replace`")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("pin remote service ls --stat", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
_, svcURL := runPinningService(t, authToken)
|
||||
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "invalid-svc", svcURL+"/invalidpath", authToken)
|
||||
|
||||
res := node.IPFS("pin", "remote", "service", "ls", "--stat")
|
||||
assert.Contains(t, res.Stdout.String(), " 0/0/0/0")
|
||||
|
||||
stats := node.IPFS("pin", "remote", "service", "ls", "--stat", "--enc=json").Stdout.String()
|
||||
assert.Equal(t, "valid", gjson.Get(stats, `RemoteServices.#(Service == "svc").Stat.Status`).Str)
|
||||
assert.Equal(t, "invalid", gjson.Get(stats, `RemoteServices.#(Service == "invalid-svc").Stat.Status`).Str)
|
||||
|
||||
// no --stat returns no stat obj
|
||||
t.Run("no --stat returns no stat obj", func(t *testing.T) {
|
||||
res := node.IPFS("pin", "remote", "service", "ls", "--enc=json")
|
||||
assert.False(t, gjson.Get(res.Stdout.String(), `RemoteServices.#(Service == "svc").Stat`).Exists())
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("adding service with invalid URL fails", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
|
||||
res := node.RunIPFS("pin", "remote", "service", "add", "svc", "invalid-service.example.com", "key")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
|
||||
|
||||
res = node.RunIPFS("pin", "remote", "service", "add", "svc", "xyz://invalid-service.example.com", "key")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
|
||||
})
|
||||
|
||||
t.Run("unauthorized pinning service calls fail", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
_, svcURL := runPinningService(t, authToken)
|
||||
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, "othertoken")
|
||||
|
||||
res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "access denied")
|
||||
})
|
||||
|
||||
t.Run("pinning service calls fail when there is a wrong path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
_, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL+"/invalid-path", authToken)
|
||||
|
||||
res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "404")
|
||||
})
|
||||
|
||||
t.Run("pinning service calls fail when DNS resolution fails", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", "https://invalid-service.example.com", authToken)
|
||||
|
||||
res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "no such host")
|
||||
})
|
||||
|
||||
t.Run("pin remote service rm", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", "https://example.com", authToken)
|
||||
node.IPFS("pin", "remote", "service", "rm", "svc")
|
||||
res := node.IPFS("pin", "remote", "service", "ls")
|
||||
assert.NotContains(t, res.Stdout.String(), "svc")
|
||||
})
|
||||
|
||||
t.Run("remote pinning", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
verifyStatus := func(node *harness.Node, name, hash, status string) {
|
||||
resJSON := node.IPFS("pin", "remote", "ls",
|
||||
"--service=svc",
|
||||
"--enc=json",
|
||||
"--name="+name,
|
||||
"--status="+status,
|
||||
).Stdout.String()
|
||||
|
||||
assert.Equal(t, status, gjson.Get(resJSON, "Status").Str)
|
||||
assert.Equal(t, hash, gjson.Get(resJSON, "Cid").Str)
|
||||
assert.Equal(t, name, gjson.Get(resJSON, "Name").Str)
|
||||
}
|
||||
|
||||
t.Run("'ipfs pin remote add --background=true'", func(t *testing.T) {
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
// retain a ptr to the pin that's in the DB so we can directly mutate its status
|
||||
// to simulate async work
|
||||
pinCh := make(chan *pinningservice.PinStatus, 1)
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pinCh <- pin
|
||||
}
|
||||
|
||||
hash := node.IPFSAddStr("foo")
|
||||
node.IPFS("pin", "remote", "add",
|
||||
"--background=true",
|
||||
"--service=svc",
|
||||
"--name=pin1",
|
||||
hash,
|
||||
)
|
||||
|
||||
pin := <-pinCh
|
||||
|
||||
transitionStatus := func(status string) {
|
||||
pin.M.Lock()
|
||||
pin.Status = status
|
||||
pin.M.Unlock()
|
||||
}
|
||||
|
||||
verifyStatus(node, "pin1", hash, "queued")
|
||||
|
||||
transitionStatus("pinning")
|
||||
verifyStatus(node, "pin1", hash, "pinning")
|
||||
|
||||
transitionStatus("pinned")
|
||||
verifyStatus(node, "pin1", hash, "pinned")
|
||||
|
||||
transitionStatus("failed")
|
||||
verifyStatus(node, "pin1", hash, "failed")
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote add --background=false'", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pin.Status = "pinned"
|
||||
}
|
||||
hash := node.IPFSAddStr("foo")
|
||||
node.IPFS("pin", "remote", "add",
|
||||
"--background=false",
|
||||
"--service=svc",
|
||||
"--name=pin2",
|
||||
hash,
|
||||
)
|
||||
verifyStatus(node, "pin2", hash, "pinned")
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote ls' with multiple statuses", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
hash := node.IPFSAddStr("foo")
|
||||
desiredStatuses := map[string]string{
|
||||
"pin-queued": "queued",
|
||||
"pin-pinning": "pinning",
|
||||
"pin-pinned": "pinned",
|
||||
"pin-failed": "failed",
|
||||
}
|
||||
var pins []*pinningservice.PinStatus
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pins = append(pins, pin)
|
||||
// this must be "pinned" for the 'pin remote add' command to return
|
||||
// after 'pin remote add', we change the status to its real status
|
||||
pin.Status = "pinned"
|
||||
}
|
||||
|
||||
for pinName := range desiredStatuses {
|
||||
node.IPFS("pin", "remote", "add",
|
||||
"--service=svc",
|
||||
"--name="+pinName,
|
||||
hash,
|
||||
)
|
||||
}
|
||||
for _, pin := range pins {
|
||||
pin.M.Lock()
|
||||
pin.Status = desiredStatuses[pin.Pin.Name]
|
||||
pin.M.Unlock()
|
||||
}
|
||||
|
||||
res := node.IPFS("pin", "remote", "ls",
|
||||
"--service=svc",
|
||||
"--status=queued,pinning,pinned,failed",
|
||||
"--enc=json",
|
||||
)
|
||||
actualStatuses := map[string]string{}
|
||||
for _, line := range res.Stdout.Lines() {
|
||||
name := gjson.Get(line, "Name").Str
|
||||
status := gjson.Get(line, "Status").Str
|
||||
// drop statuses of other pins we didn't add
|
||||
if _, ok := desiredStatuses[name]; ok {
|
||||
actualStatuses[name] = status
|
||||
}
|
||||
}
|
||||
assert.Equal(t, desiredStatuses, actualStatuses)
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote ls' by CID", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
transitionedCh := make(chan struct{}, 1)
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pin.Status = "pinned"
|
||||
transitionedCh <- struct{}{}
|
||||
}
|
||||
hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
|
||||
node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", hash)
|
||||
<-transitionedCh
|
||||
res := node.IPFS("pin", "remote", "ls", "--service=svc", "--cid="+hash, "--enc=json").Stdout.String()
|
||||
assert.Contains(t, res, hash)
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote rm --name' without --force when multiple pins match", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pin.Status = "pinned"
|
||||
}
|
||||
hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
|
||||
node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
|
||||
node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
|
||||
|
||||
t.Run("fails", func(t *testing.T) {
|
||||
res := node.RunIPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name")
|
||||
assert.Equal(t, 1, res.ExitCode())
|
||||
assert.Contains(t, res.Stderr.String(), "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal")
|
||||
})
|
||||
|
||||
t.Run("matching pins are not removed", func(t *testing.T) {
|
||||
lines := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Lines()
|
||||
assert.Contains(t, lines[0], "force-test-name")
|
||||
assert.Contains(t, lines[1], "force-test-name")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote rm --name --force' remove multiple pins", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pin.Status = "pinned"
|
||||
}
|
||||
hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
|
||||
node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
|
||||
node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
|
||||
|
||||
node.IPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name", "--force")
|
||||
out := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Trimmed()
|
||||
assert.Empty(t, out)
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote rm --force' removes all pins", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
||||
svc, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
|
||||
pin.M.Lock()
|
||||
defer pin.M.Unlock()
|
||||
pin.Status = "pinned"
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
|
||||
name := fmt.Sprintf("--name=%d", i)
|
||||
node.IPFS("pin", "remote", "add", "--service=svc", "--name="+name, hash)
|
||||
}
|
||||
|
||||
lines := node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
|
||||
assert.Len(t, lines, 4)
|
||||
|
||||
node.IPFS("pin", "remote", "rm", "--service=svc", "--force")
|
||||
|
||||
lines = node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
|
||||
assert.Len(t, lines, 0)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("'ipfs pin remote add' shows a warning message when offline", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
node := harness.NewT(t).NewNode().Init()
|
||||
_, svcURL := runPinningService(t, authToken)
|
||||
node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
|
||||
|
||||
hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
|
||||
res := node.IPFS("pin", "remote", "add", "--service=svc", "--background", hash)
|
||||
warningMsg := "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID"
|
||||
assert.Contains(t, res.Stdout.String(), warningMsg)
|
||||
})
|
||||
}
|
||||
401
test/cli/testutils/pinningservice/pinning.go
Normal file
401
test/cli/testutils/pinningservice/pinning.go
Normal file
@ -0,0 +1,401 @@
|
||||
package pinningservice
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
func NewRouter(authToken string, svc *PinningService) http.Handler {
|
||||
router := httprouter.New()
|
||||
router.GET("/api/v1/pins", svc.listPins)
|
||||
router.POST("/api/v1/pins", svc.addPin)
|
||||
router.GET("/api/v1/pins/:requestID", svc.getPin)
|
||||
router.POST("/api/v1/pins/:requestID", svc.replacePin)
|
||||
router.DELETE("/api/v1/pins/:requestID", svc.removePin)
|
||||
|
||||
handler := authHandler(authToken, router)
|
||||
|
||||
return handler
|
||||
}
|
||||
|
||||
func authHandler(authToken string, delegate http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
authz := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(authz, "Bearer ") {
|
||||
errResp(w, "invalid authorization token, must start with 'Bearer '", "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
token := strings.TrimPrefix(authz, "Bearer ")
|
||||
if token != authToken {
|
||||
errResp(w, "access denied", "", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
delegate.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func New() *PinningService {
|
||||
return &PinningService{
|
||||
PinAdded: func(*AddPinRequest, *PinStatus) {},
|
||||
}
|
||||
}
|
||||
|
||||
// PinningService is a basic pinning service that implements the Remote Pinning API, for testing Kubo's integration with remote pinning services.
|
||||
// Pins are not persisted, they are just kept in-memory, and this provides callbacks for controlling the behavior of the pinning service.
|
||||
type PinningService struct {
|
||||
m sync.Mutex
|
||||
// PinAdded is a callback that is invoked after a new pin is added via the API.
|
||||
PinAdded func(*AddPinRequest, *PinStatus)
|
||||
pins []*PinStatus
|
||||
}
|
||||
|
||||
type Pin struct {
|
||||
CID string `json:"cid"`
|
||||
Name string `json:"name"`
|
||||
Origins []string `json:"origins"`
|
||||
Meta map[string]interface{} `json:"meta"`
|
||||
}
|
||||
|
||||
type PinStatus struct {
|
||||
M sync.Mutex
|
||||
RequestID string
|
||||
Status string
|
||||
Created time.Time
|
||||
Pin Pin
|
||||
Delegates []string
|
||||
Info map[string]interface{}
|
||||
}
|
||||
|
||||
func (p *PinStatus) MarshalJSON() ([]byte, error) {
|
||||
type pinStatusJSON struct {
|
||||
RequestID string `json:"requestid"`
|
||||
Status string `json:"status"`
|
||||
Created time.Time `json:"created"`
|
||||
Pin Pin `json:"pin"`
|
||||
Delegates []string `json:"delegates"`
|
||||
Info map[string]interface{} `json:"info"`
|
||||
}
|
||||
// lock the pin before marshaling it to protect against data races while marshaling
|
||||
p.M.Lock()
|
||||
pinJSON := pinStatusJSON{
|
||||
RequestID: p.RequestID,
|
||||
Status: p.Status,
|
||||
Created: p.Created,
|
||||
Pin: p.Pin,
|
||||
Delegates: p.Delegates,
|
||||
Info: p.Info,
|
||||
}
|
||||
p.M.Unlock()
|
||||
return json.Marshal(pinJSON)
|
||||
}
|
||||
|
||||
func (p *PinStatus) Clone() PinStatus {
|
||||
return PinStatus{
|
||||
RequestID: p.RequestID,
|
||||
Status: p.Status,
|
||||
Created: p.Created,
|
||||
Pin: p.Pin,
|
||||
Delegates: p.Delegates,
|
||||
Info: p.Info,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
matchExact = "exact"
|
||||
matchIExact = "iexact"
|
||||
matchPartial = "partial"
|
||||
matchIPartial = "ipartial"
|
||||
|
||||
statusQueued = "queued"
|
||||
statusPinning = "pinning"
|
||||
statusPinned = "pinned"
|
||||
statusFailed = "failed"
|
||||
|
||||
timeLayout = "2006-01-02T15:04:05.999Z"
|
||||
)
|
||||
|
||||
func errResp(w http.ResponseWriter, reason, details string, statusCode int) {
|
||||
type errorObj struct {
|
||||
Reason string `json:"reason"`
|
||||
Details string `json:"details"`
|
||||
}
|
||||
type errorResp struct {
|
||||
Error errorObj `json:"error"`
|
||||
}
|
||||
resp := errorResp{
|
||||
Error: errorObj{
|
||||
Reason: reason,
|
||||
Details: details,
|
||||
},
|
||||
}
|
||||
writeJSON(w, resp, statusCode)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, val any, statusCode int) {
|
||||
b, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
errResp(w, fmt.Sprintf("marshaling response: %s", err), "", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(statusCode)
|
||||
_, _ = w.Write(b)
|
||||
}
|
||||
|
||||
type AddPinRequest struct {
|
||||
CID string `json:"cid"`
|
||||
Name string `json:"name"`
|
||||
Origins []string `json:"origins"`
|
||||
Meta map[string]interface{} `json:"meta"`
|
||||
}
|
||||
|
||||
func (p *PinningService) addPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
var addReq AddPinRequest
|
||||
err := json.NewDecoder(req.Body).Decode(&addReq)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("unmarshaling req: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
pin := &PinStatus{
|
||||
RequestID: uuid.NewString(),
|
||||
Status: statusQueued,
|
||||
Created: time.Now(),
|
||||
Pin: Pin(addReq),
|
||||
}
|
||||
|
||||
p.m.Lock()
|
||||
p.pins = append(p.pins, pin)
|
||||
p.m.Unlock()
|
||||
|
||||
writeJSON(writer, &pin, http.StatusAccepted)
|
||||
p.PinAdded(&addReq, pin)
|
||||
}
|
||||
|
||||
type ListPinsResponse struct {
|
||||
Count int `json:"count"`
|
||||
Results []*PinStatus `json:"results"`
|
||||
}
|
||||
|
||||
func (p *PinningService) listPins(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
q := req.URL.Query()
|
||||
|
||||
cidStr := q.Get("cid")
|
||||
name := q.Get("name")
|
||||
match := q.Get("match")
|
||||
status := q.Get("status")
|
||||
beforeStr := q.Get("before")
|
||||
afterStr := q.Get("after")
|
||||
limitStr := q.Get("limit")
|
||||
metaStr := q.Get("meta")
|
||||
|
||||
if limitStr == "" {
|
||||
limitStr = "10"
|
||||
}
|
||||
limit, err := strconv.Atoi(limitStr)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("parsing limit: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var cids []string
|
||||
if cidStr != "" {
|
||||
cids = strings.Split(cidStr, ",")
|
||||
}
|
||||
|
||||
var statuses []string
|
||||
if status != "" {
|
||||
statuses = strings.Split(status, ",")
|
||||
}
|
||||
|
||||
p.m.Lock()
|
||||
defer p.m.Unlock()
|
||||
var pins []*PinStatus
|
||||
for _, pinStatus := range p.pins {
|
||||
// clone it so we can immediately release the lock
|
||||
pinStatus.M.Lock()
|
||||
clonedPS := pinStatus.Clone()
|
||||
pinStatus.M.Unlock()
|
||||
|
||||
// cid
|
||||
var matchesCID bool
|
||||
if len(cids) == 0 {
|
||||
matchesCID = true
|
||||
} else {
|
||||
for _, cid := range cids {
|
||||
if cid == clonedPS.Pin.CID {
|
||||
matchesCID = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matchesCID {
|
||||
continue
|
||||
}
|
||||
|
||||
// name
|
||||
if match == "" {
|
||||
match = matchExact
|
||||
}
|
||||
if name != "" {
|
||||
switch match {
|
||||
case matchExact:
|
||||
if name != clonedPS.Pin.Name {
|
||||
continue
|
||||
}
|
||||
case matchIExact:
|
||||
if !strings.EqualFold(name, clonedPS.Pin.Name) {
|
||||
continue
|
||||
}
|
||||
case matchPartial:
|
||||
if !strings.Contains(clonedPS.Pin.Name, name) {
|
||||
continue
|
||||
}
|
||||
case matchIPartial:
|
||||
if !strings.Contains(strings.ToLower(clonedPS.Pin.Name), strings.ToLower(name)) {
|
||||
continue
|
||||
}
|
||||
default:
|
||||
errResp(writer, fmt.Sprintf("unknown match %q", match), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// status
|
||||
var matchesStatus bool
|
||||
if len(statuses) == 0 {
|
||||
statuses = []string{statusPinned}
|
||||
}
|
||||
for _, status := range statuses {
|
||||
if status == clonedPS.Status {
|
||||
matchesStatus = true
|
||||
}
|
||||
}
|
||||
if !matchesStatus {
|
||||
continue
|
||||
}
|
||||
|
||||
// before
|
||||
if beforeStr != "" {
|
||||
before, err := time.Parse(timeLayout, beforeStr)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !clonedPS.Created.Before(before) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// after
|
||||
if afterStr != "" {
|
||||
after, err := time.Parse(timeLayout, afterStr)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !clonedPS.Created.After(after) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// meta
|
||||
if metaStr != "" {
|
||||
meta := map[string]interface{}{}
|
||||
err := json.Unmarshal([]byte(metaStr), &meta)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("parsing meta: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var matchesMeta bool
|
||||
for k, v := range meta {
|
||||
pinV, contains := clonedPS.Pin.Meta[k]
|
||||
if !contains || !reflect.DeepEqual(pinV, v) {
|
||||
matchesMeta = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matchesMeta {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// add the original pin status, not the cloned one
|
||||
pins = append(pins, pinStatus)
|
||||
|
||||
if len(pins) == limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
out := ListPinsResponse{
|
||||
Count: len(pins),
|
||||
Results: pins,
|
||||
}
|
||||
writeJSON(writer, out, http.StatusOK)
|
||||
}
|
||||
|
||||
func (p *PinningService) getPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
requestID := params.ByName("requestID")
|
||||
p.m.Lock()
|
||||
defer p.m.Unlock()
|
||||
for _, pin := range p.pins {
|
||||
if pin.RequestID == requestID {
|
||||
writeJSON(writer, pin, http.StatusOK)
|
||||
return
|
||||
}
|
||||
}
|
||||
errResp(writer, "", "", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (p *PinningService) replacePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
requestID := params.ByName("requestID")
|
||||
|
||||
var replaceReq Pin
|
||||
err := json.NewDecoder(req.Body).Decode(&replaceReq)
|
||||
if err != nil {
|
||||
errResp(writer, fmt.Sprintf("decoding request: %s", err), "", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p.m.Lock()
|
||||
defer p.m.Unlock()
|
||||
for _, pin := range p.pins {
|
||||
if pin.RequestID == requestID {
|
||||
pin.M.Lock()
|
||||
pin.Pin = replaceReq
|
||||
pin.M.Unlock()
|
||||
writer.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
}
|
||||
errResp(writer, "", "", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (p *PinningService) removePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
requestID := params.ByName("requestID")
|
||||
|
||||
p.m.Lock()
|
||||
defer p.m.Unlock()
|
||||
|
||||
for i, pin := range p.pins {
|
||||
if pin.RequestID == requestID {
|
||||
p.pins = append(p.pins[0:i], p.pins[i+1:]...)
|
||||
writer.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
errResp(writer, "", "", http.StatusNotFound)
|
||||
}
|
||||
@ -1,332 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
test_description="Test ipfs remote pinning operations"
|
||||
|
||||
. lib/test-lib.sh
|
||||
|
||||
if [ -z ${TEST_DOCKER_HOST+x} ]; then
|
||||
# TODO: set up instead of skipping?
|
||||
skip_all='Skipping pinning service integration tests: missing TEST_DOCKER_HOST, remote pinning service not available'
|
||||
test_done
|
||||
fi
|
||||
|
||||
# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work
|
||||
test_init_ipfs
|
||||
test_launch_ipfs_daemon
|
||||
|
||||
# create user on pinning service
|
||||
TEST_PIN_SVC="http://${TEST_DOCKER_HOST}:5000/api/v1"
|
||||
TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token)
|
||||
|
||||
# pin remote service add|ls|rm
|
||||
|
||||
# confirm empty service list response has proper json struct
|
||||
# https://github.com/ipfs/go-ipfs/pull/7829
|
||||
test_expect_success "test 'ipfs pin remote service ls' JSON on empty list" '
|
||||
ipfs pin remote service ls --stat --enc=json | tee empty_ls_out &&
|
||||
echo "{\"RemoteServices\":[]}" > exp_ls_out &&
|
||||
test_cmp exp_ls_out empty_ls_out
|
||||
'
|
||||
|
||||
# add valid and invalid services
|
||||
test_expect_success "creating test user on remote pinning service" '
|
||||
echo CI host IP address ${TEST_PIN_SVC} &&
|
||||
ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} &&
|
||||
ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key &&
|
||||
ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key &&
|
||||
ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key &&
|
||||
ipfs pin remote service add test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY}
|
||||
'
|
||||
|
||||
# add a service with a invalid endpoint
|
||||
test_expect_success "adding remote service with invalid endpoint" '
|
||||
test_expect_code 1 ipfs pin remote service add test_endpoint_no_protocol invalid-service.example.com fake_api_key &&
|
||||
test_expect_code 1 ipfs pin remote service add test_endpoint_bad_protocol xyz://invalid-service.example.com fake_api_key
|
||||
'
|
||||
|
||||
test_expect_success "test 'ipfs pin remote service ls'" '
|
||||
ipfs pin remote service ls | tee ls_out &&
|
||||
grep -q test_pin_svc ls_out &&
|
||||
grep -q test_invalid_key_svc ls_out &&
|
||||
grep -q test_invalid_url_path_svc ls_out &&
|
||||
grep -q test_invalid_url_dns_svc ls_out
|
||||
'
|
||||
|
||||
test_expect_success "test enabling mfs pinning" '
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval \"10s\" &&
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName \"mfs_test_pin\" &&
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable true &&
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval > repin_interval &&
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName > pin_name &&
|
||||
ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable > enable &&
|
||||
echo 10s > expected_repin_interval &&
|
||||
echo mfs_test_pin > expected_pin_name &&
|
||||
echo true > expected_enable &&
|
||||
test_cmp repin_interval expected_repin_interval &&
|
||||
test_cmp pin_name expected_pin_name &&
|
||||
test_cmp enable expected_enable
|
||||
'
|
||||
|
||||
# expect PIN to be created
|
||||
test_expect_success "verify MFS root is being pinned" '
|
||||
ipfs files cp /ipfs/bafkqaaa /mfs-pinning-test-$(date +%s.%N) &&
|
||||
ipfs files flush &&
|
||||
sleep 31 &&
|
||||
ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
|
||||
ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
|
||||
cat mfs_cid ls_out &&
|
||||
test_cmp mfs_cid pin_cid
|
||||
'
|
||||
|
||||
# expect existing PIN to be replaced
|
||||
test_expect_success "verify MFS root is being repinned on CID change" '
|
||||
ipfs files cp /ipfs/bafkqaaa /mfs-pinning-repin-test-$(date +%s.%N) &&
|
||||
ipfs files flush &&
|
||||
sleep 31 &&
|
||||
ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
|
||||
ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
|
||||
cat mfs_cid ls_out &&
|
||||
test_cmp mfs_cid pin_cid
|
||||
'
|
||||
|
||||
# SECURITY of access tokens in API.Key fields:
|
||||
# Pinning.RemoteServices includes API.Key, and we give it the same treatment
|
||||
# as Identity.PrivKey to prevent exposing it on the network
|
||||
|
||||
test_expect_success "'ipfs config Pinning' fails" '
|
||||
test_expect_code 1 ipfs config Pinning 2>&1 > config_out
|
||||
'
|
||||
test_expect_success "output does not include API.Key" '
|
||||
test_expect_code 1 grep -q Key config_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.API.Key' fails" '
|
||||
test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.API.Key 2> config_out
|
||||
'
|
||||
|
||||
test_expect_success "output includes meaningful error" '
|
||||
echo "Error: cannot show or change pinning services credentials" > config_exp &&
|
||||
test_cmp config_exp config_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc' fails" '
|
||||
test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc 2> config_out
|
||||
'
|
||||
test_expect_success "output includes meaningful error" '
|
||||
test_cmp config_exp config_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs config show' does not include Pinning.RemoteServices[*].API.Key" '
|
||||
ipfs config show | tee show_config | jq -r .Pinning.RemoteServices > remote_services &&
|
||||
test_expect_code 1 grep \"Key\" remote_services &&
|
||||
test_expect_code 1 grep fake_api_key show_config &&
|
||||
test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs config replace' injects Pinning.RemoteServices[*].API.Key back" '
|
||||
test_expect_code 1 grep fake_api_key show_config &&
|
||||
test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config &&
|
||||
ipfs config replace show_config &&
|
||||
test_expect_code 0 grep fake_api_key "$IPFS_PATH/config" &&
|
||||
test_expect_code 0 grep "$TEST_PIN_SVC_KEY" "$IPFS_PATH/config"
|
||||
'
|
||||
|
||||
# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
|
||||
test_expect_success "'ipfs config replace' with Pinning.RemoteServices[*].API.Key errors out" '
|
||||
jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"myservice\": {\"API\": {\"Endpoint\": \"https://example.com/psa\", \"Key\": \"mysecret\"}}}}" > new_config &&
|
||||
test_expect_code 1 ipfs config replace - < new_config 2> replace_out
|
||||
'
|
||||
test_expect_success "output includes meaningful error" "
|
||||
echo \"Error: cannot add or remove remote pinning services with 'config replace'\" > replace_expected &&
|
||||
test_cmp replace_out replace_expected
|
||||
"
|
||||
|
||||
# /SECURITY
|
||||
|
||||
test_expect_success "pin remote service ls --stat' returns numbers for a valid service" '
|
||||
ipfs pin remote service ls --stat | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$"
|
||||
'
|
||||
|
||||
test_expect_success "pin remote service ls --enc=json --stat' returns valid status" "
|
||||
ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_pin_svc\") | .Stat.Status' | tee stat_out &&
|
||||
echo valid > stat_expected &&
|
||||
test_cmp stat_out stat_expected
|
||||
"
|
||||
|
||||
test_expect_success "pin remote service ls --stat' returns invalid status for invalid service" '
|
||||
ipfs pin remote service ls --stat | grep -E "^test_invalid_url_path_svc.+invalid$"
|
||||
'
|
||||
|
||||
test_expect_success "pin remote service ls --enc=json --stat' returns invalid status" "
|
||||
ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out &&
|
||||
echo invalid > stat_expected &&
|
||||
test_cmp stat_out stat_expected
|
||||
"
|
||||
|
||||
test_expect_success "pin remote service ls --enc=json' (without --stat) returns no Stat object" "
|
||||
ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat' | tee stat_out &&
|
||||
echo null > stat_expected &&
|
||||
test_cmp stat_out stat_expected
|
||||
"
|
||||
|
||||
test_expect_success "check connection to the test pinning service" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json
|
||||
'
|
||||
|
||||
test_expect_success "unauthorized pinning service calls fail" '
|
||||
test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc
|
||||
'
|
||||
|
||||
test_expect_success "misconfigured pinning service calls fail (wrong path)" '
|
||||
test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc
|
||||
'
|
||||
|
||||
test_expect_success "misconfigured pinning service calls fail (dns error)" '
|
||||
test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc
|
||||
'
|
||||
|
||||
# pin remote service rm
|
||||
|
||||
test_expect_success "remove pinning service" '
|
||||
ipfs pin remote service rm test_invalid_key_svc &&
|
||||
ipfs pin remote service rm test_invalid_url_path_svc &&
|
||||
ipfs pin remote service rm test_invalid_url_dns_svc
|
||||
'
|
||||
|
||||
test_expect_success "verify pinning service removal works" '
|
||||
ipfs pin remote service ls | tee ls_out &&
|
||||
test_expect_code 1 grep test_invalid_key_svc ls_out &&
|
||||
test_expect_code 1 grep test_invalid_url_path_svc ls_out &&
|
||||
test_expect_code 1 grep test_invalid_url_dns_svc ls_out
|
||||
'
|
||||
|
||||
# pin remote add
|
||||
|
||||
# we leverage the fact that inlined CID can be pinned instantly on the remote service
|
||||
# (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8)
|
||||
# below test ensures that assumption is correct (before we proceed to actual tests)
|
||||
test_expect_success "verify that default add (implicit --background=false) works with data inlined in CID" '
|
||||
ipfs pin remote add --service=test_pin_svc --name=inlined_null bafkqaaa &&
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=pinned | jq --raw-output .Status | tee ls_out &&
|
||||
grep -q "pinned" ls_out
|
||||
'
|
||||
|
||||
test_remote_pins() {
|
||||
BASE=$1
|
||||
if [ -n "$BASE" ]; then
|
||||
BASE_ARGS="--cid-base=$BASE"
|
||||
fi
|
||||
|
||||
# note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state
|
||||
test_expect_success "create some hashes using base $BASE" '
|
||||
export HASH_A=$(echo -n "A @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
|
||||
export HASH_B=$(echo -n "B @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
|
||||
export HASH_C=$(echo -n "C @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
|
||||
export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --only-hash) &&
|
||||
echo "A: $HASH_A" &&
|
||||
echo "B: $HASH_B" &&
|
||||
echo "C: $HASH_C" &&
|
||||
echo "M: $HASH_MISSING"
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote add --background=true'" '
|
||||
ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A
|
||||
'
|
||||
|
||||
test_expect_success "verify background add worked (instantly pinned variant)" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out &&
|
||||
test_expect_code 0 grep -q name_a ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_A ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" '
|
||||
test_expect_code 0 ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING
|
||||
'
|
||||
|
||||
test_expect_success "verify background add worked (queued variant)" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_m --status=queued,pinning | tee ls_out &&
|
||||
test_expect_code 0 grep -q name_m ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_MISSING ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote add --background=false'" '
|
||||
test_expect_code 0 ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B
|
||||
'
|
||||
|
||||
test_expect_success "verify foreground add worked" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out &&
|
||||
test_expect_code 0 grep -q name_b ls_out &&
|
||||
test_expect_code 0 grep -q pinned ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_B ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_A ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_B ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_MISSING ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote ls' for existing pins by CID" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_B ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote ls' for existing pins by name" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_A ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote ls' for ongoing pins by status" '
|
||||
ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_MISSING ls_out
|
||||
'
|
||||
|
||||
# --force is required only when more than a single match is found,
|
||||
# so we add second pin with the same name (but different CID) to simulate that scenario
|
||||
test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" '
|
||||
test_expect_code 0 ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_C &&
|
||||
test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --name=name_b 2> rm_out &&
|
||||
echo "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal" > rm_expected &&
|
||||
test_cmp rm_out rm_expected
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_B ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_C ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote rm --name' with --force removes all matching pins" '
|
||||
test_expect_code 0 ipfs pin remote rm --service=test_pin_svc --name=name_b --force &&
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out &&
|
||||
test_expect_code 1 grep -q $HASH_B ls_out &&
|
||||
test_expect_code 1 grep -q $HASH_C ls_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs pin remote rm --force' removes all pinned items" '
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_A ls_out &&
|
||||
test_expect_code 0 grep -q $HASH_MISSING ls_out &&
|
||||
ipfs pin remote rm --service=test_pin_svc --status=queued,pinning,pinned,failed --force &&
|
||||
ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out &&
|
||||
test_expect_code 1 grep -q $HASH_A ls_out &&
|
||||
test_expect_code 1 grep -q $HASH_MISSING ls_out
|
||||
'
|
||||
|
||||
}
|
||||
|
||||
test_remote_pins ""
|
||||
|
||||
test_kill_ipfs_daemon
|
||||
|
||||
WARNINGMESSAGE="WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID"
|
||||
|
||||
test_expect_success "'ipfs pin remote add' shows the warning message while offline" '
|
||||
test_expect_code 0 ipfs pin remote add --service=test_pin_svc --background $BASE_ARGS --name=name_a $HASH_A > actual &&
|
||||
test_expect_code 0 grep -q "$WARNINGMESSAGE" actual
|
||||
'
|
||||
|
||||
test_done
|
||||
|
||||
# vim: ts=2 sw=2 sts=2 et:
|
||||
Loading…
Reference in New Issue
Block a user