mirror of
https://github.com/ipfs/kubo.git
synced 2026-03-02 23:08:07 +08:00
Merge pull request #2962 from ipfs/kevina/block-rm
Add "ipfs block rm" command.
This commit is contained in:
Jeromy Johnson
GitHub
commit
10048ceca9
@ -32,7 +32,7 @@ func (b *arccache) DeleteBlock(k key.Key) error {
|
||||
switch err {
|
||||
case nil, ds.ErrNotFound, ErrNotFound:
|
||||
b.arc.Add(k, false)
|
||||
return nil
|
||||
return err
|
||||
default:
|
||||
return err
|
||||
}
|
||||
|
||||
@ -9,8 +9,11 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/go-ipfs/blocks"
|
||||
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
|
||||
key "github.com/ipfs/go-ipfs/blocks/key"
|
||||
cmds "github.com/ipfs/go-ipfs/commands"
|
||||
"github.com/ipfs/go-ipfs/pin"
|
||||
ds "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore"
|
||||
mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
|
||||
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
|
||||
)
|
||||
@ -38,6 +41,7 @@ multihash.
|
||||
"stat": blockStatCmd,
|
||||
"get": blockGetCmd,
|
||||
"put": blockPutCmd,
|
||||
"rm": blockRmCmd,
|
||||
},
|
||||
}
|
||||
|
||||
@ -185,3 +189,135 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
|
||||
log.Debugf("ipfs block: got block with key: %q", b.Key())
|
||||
return b, nil
|
||||
}
|
||||
|
||||
var blockRmCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
Tagline: "Remove IPFS block(s).",
|
||||
ShortDescription: `
|
||||
'ipfs block rm' is a plumbing command for removing raw ipfs blocks.
|
||||
It takes a list of base58 encoded multihashs to remove.
|
||||
`,
|
||||
},
|
||||
Arguments: []cmds.Argument{
|
||||
cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to remove."),
|
||||
},
|
||||
Options: []cmds.Option{
|
||||
cmds.BoolOption("force", "f", "Ignore nonexistent blocks.").Default(false),
|
||||
cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false),
|
||||
},
|
||||
Run: func(req cmds.Request, res cmds.Response) {
|
||||
n, err := req.InvocContext().GetNode()
|
||||
if err != nil {
|
||||
res.SetError(err, cmds.ErrNormal)
|
||||
return
|
||||
}
|
||||
hashes := req.Arguments()
|
||||
force, _, _ := req.Option("force").Bool()
|
||||
quiet, _, _ := req.Option("quiet").Bool()
|
||||
keys := make([]key.Key, 0, len(hashes))
|
||||
for _, hash := range hashes {
|
||||
k := key.B58KeyDecode(hash)
|
||||
keys = append(keys, k)
|
||||
}
|
||||
outChan := make(chan interface{})
|
||||
res.SetOutput((<-chan interface{})(outChan))
|
||||
go func() {
|
||||
defer close(outChan)
|
||||
pinning := n.Pinning
|
||||
err := rmBlocks(n.Blockstore, pinning, outChan, keys, rmBlocksOpts{
|
||||
quiet: quiet,
|
||||
force: force,
|
||||
})
|
||||
if err != nil {
|
||||
outChan <- &RemovedBlock{Error: err.Error()}
|
||||
}
|
||||
}()
|
||||
return
|
||||
},
|
||||
PostRun: func(req cmds.Request, res cmds.Response) {
|
||||
if res.Error() != nil {
|
||||
return
|
||||
}
|
||||
outChan, ok := res.Output().(<-chan interface{})
|
||||
if !ok {
|
||||
res.SetError(u.ErrCast(), cmds.ErrNormal)
|
||||
return
|
||||
}
|
||||
res.SetOutput(nil)
|
||||
|
||||
someFailed := false
|
||||
for out := range outChan {
|
||||
o := out.(*RemovedBlock)
|
||||
if o.Hash == "" && o.Error != "" {
|
||||
res.SetError(fmt.Errorf("aborted: %s", o.Error), cmds.ErrNormal)
|
||||
return
|
||||
} else if o.Error != "" {
|
||||
someFailed = true
|
||||
fmt.Fprintf(res.Stderr(), "cannot remove %s: %s\n", o.Hash, o.Error)
|
||||
} else {
|
||||
fmt.Fprintf(res.Stdout(), "removed %s\n", o.Hash)
|
||||
}
|
||||
}
|
||||
if someFailed {
|
||||
res.SetError(fmt.Errorf("some blocks not removed"), cmds.ErrNormal)
|
||||
}
|
||||
},
|
||||
Type: RemovedBlock{},
|
||||
}
|
||||
|
||||
type RemovedBlock struct {
|
||||
Hash string `json:",omitempty"`
|
||||
Error string `json:",omitempty"`
|
||||
}
|
||||
|
||||
type rmBlocksOpts struct {
|
||||
quiet bool
|
||||
force bool
|
||||
}
|
||||
|
||||
func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, keys []key.Key, opts rmBlocksOpts) error {
|
||||
unlocker := blocks.GCLock()
|
||||
defer unlocker.Unlock()
|
||||
|
||||
stillOkay, err := checkIfPinned(pins, keys, out)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pin check failed: %s", err)
|
||||
}
|
||||
|
||||
for _, k := range stillOkay {
|
||||
err := blocks.DeleteBlock(k)
|
||||
if err != nil && opts.force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
|
||||
// ignore non-existent blocks
|
||||
} else if err != nil {
|
||||
out <- &RemovedBlock{Hash: k.String(), Error: err.Error()}
|
||||
} else if !opts.quiet {
|
||||
out <- &RemovedBlock{Hash: k.String()}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkIfPinned(pins pin.Pinner, keys []key.Key, out chan<- interface{}) ([]key.Key, error) {
|
||||
stillOkay := make([]key.Key, 0, len(keys))
|
||||
res, err := pins.CheckIfPinned(keys...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range res {
|
||||
switch r.Mode {
|
||||
case pin.NotPinned:
|
||||
stillOkay = append(stillOkay, r.Key)
|
||||
case pin.Indirect:
|
||||
out <- &RemovedBlock{
|
||||
Hash: r.Key.String(),
|
||||
Error: fmt.Sprintf("pinned via %s", r.Via)}
|
||||
default:
|
||||
modeStr, _ := pin.PinModeToString(r.Mode)
|
||||
out <- &RemovedBlock{
|
||||
Hash: r.Key.String(),
|
||||
Error: fmt.Sprintf("pinned: %s", modeStr)}
|
||||
|
||||
}
|
||||
}
|
||||
return stillOkay, nil
|
||||
}
|
||||
|
||||
74
pin/pin.go
74
pin/pin.go
@ -75,6 +75,10 @@ type Pinner interface {
|
||||
Pin(context.Context, *mdag.Node, bool) error
|
||||
Unpin(context.Context, key.Key, bool) error
|
||||
|
||||
// Check if a set of keys are pinned, more efficient than
|
||||
// calling IsPinned for each key
|
||||
CheckIfPinned(keys ...key.Key) ([]Pinned, error)
|
||||
|
||||
// PinWithMode is for manually editing the pin structure. Use with
|
||||
// care! If used improperly, garbage collection may not be
|
||||
// successful.
|
||||
@ -90,6 +94,12 @@ type Pinner interface {
|
||||
InternalPins() []key.Key
|
||||
}
|
||||
|
||||
type Pinned struct {
|
||||
Key key.Key
|
||||
Mode PinMode
|
||||
Via key.Key
|
||||
}
|
||||
|
||||
// pinner implements the Pinner interface
|
||||
type pinner struct {
|
||||
lock sync.RWMutex
|
||||
@ -255,6 +265,70 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func (p *pinner) CheckIfPinned(keys ...key.Key) ([]Pinned, error) {
|
||||
p.lock.RLock()
|
||||
defer p.lock.RUnlock()
|
||||
pinned := make([]Pinned, 0, len(keys))
|
||||
toCheck := make(map[key.Key]struct{})
|
||||
|
||||
// First check for non-Indirect pins directly
|
||||
for _, k := range keys {
|
||||
if p.recursePin.HasKey(k) {
|
||||
pinned = append(pinned, Pinned{Key: k, Mode: Recursive})
|
||||
} else if p.directPin.HasKey(k) {
|
||||
pinned = append(pinned, Pinned{Key: k, Mode: Direct})
|
||||
} else if p.isInternalPin(k) {
|
||||
pinned = append(pinned, Pinned{Key: k, Mode: Internal})
|
||||
} else {
|
||||
toCheck[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Now walk all recursive pins to check for indirect pins
|
||||
var checkChildren func(key.Key, key.Key) error
|
||||
checkChildren = func(rk key.Key, parentKey key.Key) error {
|
||||
parent, err := p.dserv.Get(context.Background(), parentKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lnk := range parent.Links {
|
||||
k := key.Key(lnk.Hash)
|
||||
|
||||
if _, found := toCheck[k]; found {
|
||||
pinned = append(pinned,
|
||||
Pinned{Key: k, Mode: Indirect, Via: rk})
|
||||
delete(toCheck, k)
|
||||
}
|
||||
|
||||
err := checkChildren(rk, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(toCheck) == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, rk := range p.recursePin.GetKeys() {
|
||||
err := checkChildren(rk, rk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(toCheck) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Anything left in toCheck is not pinned
|
||||
for k, _ := range toCheck {
|
||||
pinned = append(pinned, Pinned{Key: k, Mode: NotPinned})
|
||||
}
|
||||
|
||||
return pinned, nil
|
||||
}
|
||||
|
||||
func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
|
||||
p.lock.Lock()
|
||||
defer p.lock.Unlock()
|
||||
|
||||
@ -10,17 +10,26 @@ test_description="Test block command"
|
||||
|
||||
test_init_ipfs
|
||||
|
||||
HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk"
|
||||
|
||||
#
|
||||
# "block put tests"
|
||||
#
|
||||
|
||||
test_expect_success "'ipfs block put' succeeds" '
|
||||
echo "Hello Mars!" >expected_in &&
|
||||
ipfs block put <expected_in >actual_out
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs block put' output looks good" '
|
||||
HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk" &&
|
||||
echo "$HASH" >expected_out &&
|
||||
test_cmp expected_out actual_out
|
||||
'
|
||||
|
||||
#
|
||||
# "block get" tests
|
||||
#
|
||||
|
||||
test_expect_success "'ipfs block get' succeeds" '
|
||||
ipfs block get $HASH >actual_in
|
||||
'
|
||||
@ -29,16 +38,141 @@ test_expect_success "'ipfs block get' output looks good" '
|
||||
test_cmp expected_in actual_in
|
||||
'
|
||||
|
||||
#
|
||||
# "block stat" tests
|
||||
#
|
||||
|
||||
test_expect_success "'ipfs block stat' succeeds" '
|
||||
ipfs block stat $HASH >actual_stat
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs block get' output looks good" '
|
||||
test_expect_success "'ipfs block stat' output looks good" '
|
||||
echo "Key: $HASH" >expected_stat &&
|
||||
echo "Size: 12" >>expected_stat &&
|
||||
test_cmp expected_stat actual_stat
|
||||
'
|
||||
|
||||
#
|
||||
# "block rm" tests
|
||||
#
|
||||
|
||||
test_expect_success "'ipfs block rm' succeeds" '
|
||||
ipfs block rm $HASH >actual_rm
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs block rm' output looks good" '
|
||||
echo "removed $HASH" > expected_rm &&
|
||||
test_cmp expected_rm actual_rm
|
||||
'
|
||||
|
||||
test_expect_success "'ipfs block rm' block actually removed" '
|
||||
test_must_fail ipfs block stat $HASH
|
||||
'
|
||||
|
||||
DIRHASH=QmdWmVmM6W2abTgkEfpbtA1CJyTWS2rhuUB9uP1xV8Uwtf
|
||||
FILE1HASH=Qmae3RedM7SNkWGsdzYzsr6svmsFdsva4WoTvYYsWhUSVz
|
||||
FILE2HASH=QmUtkGLvPf63NwVzLPKPUYgwhn8ZYPWF6vKWN3fZ2amfJF
|
||||
FILE3HASH=Qmesmmf1EEG1orJb6XdK6DabxexsseJnCfw8pqWgonbkoj
|
||||
|
||||
test_expect_success "add and pin directory" '
|
||||
mkdir adir &&
|
||||
echo "file1" > adir/file1 &&
|
||||
echo "file2" > adir/file2 &&
|
||||
echo "file3" > adir/file3 &&
|
||||
ipfs add -r adir
|
||||
ipfs pin add -r $DIRHASH
|
||||
'
|
||||
|
||||
test_expect_success "can't remove pinned block" '
|
||||
test_must_fail ipfs block rm $DIRHASH 2> block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "can't remove pinned block: output looks good" '
|
||||
grep -q "$DIRHASH: pinned: recursive" block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "can't remove indirectly pinned block" '
|
||||
test_must_fail ipfs block rm $FILE1HASH 2> block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "can't remove indirectly pinned block: output looks good" '
|
||||
grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "remove pin" '
|
||||
ipfs pin rm -r $DIRHASH
|
||||
'
|
||||
|
||||
test_expect_success "multi-block 'ipfs block rm' succeeds" '
|
||||
ipfs block rm $FILE1HASH $FILE2HASH $FILE3HASH > actual_rm
|
||||
'
|
||||
|
||||
test_expect_success "multi-block 'ipfs block rm' output looks good" '
|
||||
grep -F -q "removed $FILE1HASH" actual_rm &&
|
||||
grep -F -q "removed $FILE2HASH" actual_rm &&
|
||||
grep -F -q "removed $FILE3HASH" actual_rm
|
||||
'
|
||||
|
||||
test_expect_success "'add some blocks' succeeds" '
|
||||
echo "Hello Mars!" | ipfs block put &&
|
||||
echo "Hello Venus!" | ipfs block put
|
||||
'
|
||||
|
||||
test_expect_success "add and pin directory" '
|
||||
ipfs add -r adir
|
||||
ipfs pin add -r $DIRHASH
|
||||
'
|
||||
|
||||
HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk
|
||||
HASH2=QmdnpnsaEj69isdw5sNzp3h3HkaDz7xKq7BmvFFBzNr5e7
|
||||
RANDOMHASH=QRmKqGMAM6EbngbZjSqrvYzq5Qd8b1bSWymjSUY9zQSNDq
|
||||
|
||||
test_expect_success "multi-block 'ipfs block rm' mixed" '
|
||||
test_must_fail ipfs block rm $FILE1HASH $DIRHASH $HASH $FILE3HASH $RANDOMHASH $HASH2 2> block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "pinned block not removed" '
|
||||
ipfs block stat $FILE1HASH &&
|
||||
ipfs block stat $FILE3HASH
|
||||
'
|
||||
|
||||
test_expect_success "non-pinned blocks removed" '
|
||||
test_must_fail ipfs block stat $HASH &&
|
||||
test_must_fail ipfs block stat $HASH2
|
||||
'
|
||||
|
||||
test_expect_success "error reported on removing non-existent block" '
|
||||
grep -q "cannot remove $RANDOMHASH" block_rm_err
|
||||
'
|
||||
|
||||
test_expect_success "'add some blocks' succeeds" '
|
||||
echo "Hello Mars!" | ipfs block put &&
|
||||
echo "Hello Venus!" | ipfs block put
|
||||
'
|
||||
|
||||
test_expect_success "multi-block 'ipfs block rm -f' with non existent blocks succeed" '
|
||||
ipfs block rm -f $HASH $RANDOMHASH $HASH2
|
||||
'
|
||||
|
||||
test_expect_success "existent blocks removed" '
|
||||
test_must_fail ipfs block stat $HASH &&
|
||||
test_must_fail ipfs block stat $HASH2
|
||||
'
|
||||
|
||||
test_expect_success "'add some blocks' succeeds" '
|
||||
echo "Hello Mars!" | ipfs block put &&
|
||||
echo "Hello Venus!" | ipfs block put
|
||||
'
|
||||
|
||||
test_expect_success "multi-block 'ipfs block rm -q' produces no output" '
|
||||
ipfs block rm -q $HASH $HASH2 > block_rm_out &&
|
||||
test ! -s block_rm_out
|
||||
'
|
||||
|
||||
#
|
||||
# Misc tests
|
||||
#
|
||||
|
||||
test_expect_success "'ipfs block stat' with nothing from stdin doesnt crash" '
|
||||
test_expect_code 1 ipfs block stat < /dev/null 2> stat_out
|
||||
'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user