diff --git a/blocks/blockstore/arc_cache.go b/blocks/blockstore/arc_cache.go index 63253ef9c..2b6aa04e2 100644 --- a/blocks/blockstore/arc_cache.go +++ b/blocks/blockstore/arc_cache.go @@ -32,7 +32,7 @@ func (b *arccache) DeleteBlock(k key.Key) error { switch err { case nil, ds.ErrNotFound, ErrNotFound: b.arc.Add(k, false) - return nil + return err default: return err } diff --git a/core/commands/block.go b/core/commands/block.go index 6a2ed0da8..831b662b1 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -9,8 +9,11 @@ import ( "strings" "github.com/ipfs/go-ipfs/blocks" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" key "github.com/ipfs/go-ipfs/blocks/key" cmds "github.com/ipfs/go-ipfs/commands" + "github.com/ipfs/go-ipfs/pin" + ds "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore" mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -38,6 +41,7 @@ multihash. "stat": blockStatCmd, "get": blockGetCmd, "put": blockPutCmd, + "rm": blockRmCmd, }, } @@ -185,3 +189,135 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) { log.Debugf("ipfs block: got block with key: %q", b.Key()) return b, nil } + +var blockRmCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove IPFS block(s).", + ShortDescription: ` +'ipfs block rm' is a plumbing command for removing raw ipfs blocks. +It takes a list of base58 encoded multihashs to remove. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to remove."), + }, + Options: []cmds.Option{ + cmds.BoolOption("force", "f", "Ignore nonexistent blocks.").Default(false), + cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + hashes := req.Arguments() + force, _, _ := req.Option("force").Bool() + quiet, _, _ := req.Option("quiet").Bool() + keys := make([]key.Key, 0, len(hashes)) + for _, hash := range hashes { + k := key.B58KeyDecode(hash) + keys = append(keys, k) + } + outChan := make(chan interface{}) + res.SetOutput((<-chan interface{})(outChan)) + go func() { + defer close(outChan) + pinning := n.Pinning + err := rmBlocks(n.Blockstore, pinning, outChan, keys, rmBlocksOpts{ + quiet: quiet, + force: force, + }) + if err != nil { + outChan <- &RemovedBlock{Error: err.Error()} + } + }() + return + }, + PostRun: func(req cmds.Request, res cmds.Response) { + if res.Error() != nil { + return + } + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + res.SetOutput(nil) + + someFailed := false + for out := range outChan { + o := out.(*RemovedBlock) + if o.Hash == "" && o.Error != "" { + res.SetError(fmt.Errorf("aborted: %s", o.Error), cmds.ErrNormal) + return + } else if o.Error != "" { + someFailed = true + fmt.Fprintf(res.Stderr(), "cannot remove %s: %s\n", o.Hash, o.Error) + } else { + fmt.Fprintf(res.Stdout(), "removed %s\n", o.Hash) + } + } + if someFailed { + res.SetError(fmt.Errorf("some blocks not removed"), cmds.ErrNormal) + } + }, + Type: RemovedBlock{}, +} + +type RemovedBlock struct { + Hash string `json:",omitempty"` + Error string `json:",omitempty"` +} + +type rmBlocksOpts struct { + quiet bool + force bool +} + +func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, keys []key.Key, opts rmBlocksOpts) error { + unlocker := blocks.GCLock() + defer unlocker.Unlock() + + stillOkay, err := checkIfPinned(pins, keys, out) + if err != nil { + return fmt.Errorf("pin check failed: %s", err) + } + + for _, k := range stillOkay { + err := blocks.DeleteBlock(k) + if err != nil && opts.force && (err == bs.ErrNotFound || err == ds.ErrNotFound) { + // ignore non-existent blocks + } else if err != nil { + out <- &RemovedBlock{Hash: k.String(), Error: err.Error()} + } else if !opts.quiet { + out <- &RemovedBlock{Hash: k.String()} + } + } + return nil +} + +func checkIfPinned(pins pin.Pinner, keys []key.Key, out chan<- interface{}) ([]key.Key, error) { + stillOkay := make([]key.Key, 0, len(keys)) + res, err := pins.CheckIfPinned(keys...) + if err != nil { + return nil, err + } + for _, r := range res { + switch r.Mode { + case pin.NotPinned: + stillOkay = append(stillOkay, r.Key) + case pin.Indirect: + out <- &RemovedBlock{ + Hash: r.Key.String(), + Error: fmt.Sprintf("pinned via %s", r.Via)} + default: + modeStr, _ := pin.PinModeToString(r.Mode) + out <- &RemovedBlock{ + Hash: r.Key.String(), + Error: fmt.Sprintf("pinned: %s", modeStr)} + + } + } + return stillOkay, nil +} diff --git a/pin/pin.go b/pin/pin.go index fa2af39db..49c5a8dd1 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -75,6 +75,10 @@ type Pinner interface { Pin(context.Context, *mdag.Node, bool) error Unpin(context.Context, key.Key, bool) error + // Check if a set of keys are pinned, more efficient than + // calling IsPinned for each key + CheckIfPinned(keys ...key.Key) ([]Pinned, error) + // PinWithMode is for manually editing the pin structure. Use with // care! If used improperly, garbage collection may not be // successful. @@ -90,6 +94,12 @@ type Pinner interface { InternalPins() []key.Key } +type Pinned struct { + Key key.Key + Mode PinMode + Via key.Key +} + // pinner implements the Pinner interface type pinner struct { lock sync.RWMutex @@ -255,6 +265,70 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error) return "", false, nil } +func (p *pinner) CheckIfPinned(keys ...key.Key) ([]Pinned, error) { + p.lock.RLock() + defer p.lock.RUnlock() + pinned := make([]Pinned, 0, len(keys)) + toCheck := make(map[key.Key]struct{}) + + // First check for non-Indirect pins directly + for _, k := range keys { + if p.recursePin.HasKey(k) { + pinned = append(pinned, Pinned{Key: k, Mode: Recursive}) + } else if p.directPin.HasKey(k) { + pinned = append(pinned, Pinned{Key: k, Mode: Direct}) + } else if p.isInternalPin(k) { + pinned = append(pinned, Pinned{Key: k, Mode: Internal}) + } else { + toCheck[k] = struct{}{} + } + } + + // Now walk all recursive pins to check for indirect pins + var checkChildren func(key.Key, key.Key) error + checkChildren = func(rk key.Key, parentKey key.Key) error { + parent, err := p.dserv.Get(context.Background(), parentKey) + if err != nil { + return err + } + for _, lnk := range parent.Links { + k := key.Key(lnk.Hash) + + if _, found := toCheck[k]; found { + pinned = append(pinned, + Pinned{Key: k, Mode: Indirect, Via: rk}) + delete(toCheck, k) + } + + err := checkChildren(rk, k) + if err != nil { + return err + } + + if len(toCheck) == 0 { + return nil + } + } + return nil + } + for _, rk := range p.recursePin.GetKeys() { + err := checkChildren(rk, rk) + if err != nil { + return nil, err + } + if len(toCheck) == 0 { + break + } + } + + // Anything left in toCheck is not pinned + for k, _ := range toCheck { + pinned = append(pinned, Pinned{Key: k, Mode: NotPinned}) + } + + return pinned, nil +} + func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) { p.lock.Lock() defer p.lock.Unlock() diff --git a/test/sharness/t0050-block.sh b/test/sharness/t0050-block.sh index c4f00e048..3cb0aeaca 100755 --- a/test/sharness/t0050-block.sh +++ b/test/sharness/t0050-block.sh @@ -10,17 +10,26 @@ test_description="Test block command" test_init_ipfs +HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk" + +# +# "block put tests" +# + test_expect_success "'ipfs block put' succeeds" ' echo "Hello Mars!" >expected_in && ipfs block put actual_out ' test_expect_success "'ipfs block put' output looks good" ' - HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk" && echo "$HASH" >expected_out && test_cmp expected_out actual_out ' +# +# "block get" tests +# + test_expect_success "'ipfs block get' succeeds" ' ipfs block get $HASH >actual_in ' @@ -29,16 +38,141 @@ test_expect_success "'ipfs block get' output looks good" ' test_cmp expected_in actual_in ' +# +# "block stat" tests +# + test_expect_success "'ipfs block stat' succeeds" ' ipfs block stat $HASH >actual_stat ' -test_expect_success "'ipfs block get' output looks good" ' +test_expect_success "'ipfs block stat' output looks good" ' echo "Key: $HASH" >expected_stat && echo "Size: 12" >>expected_stat && test_cmp expected_stat actual_stat ' +# +# "block rm" tests +# + +test_expect_success "'ipfs block rm' succeeds" ' + ipfs block rm $HASH >actual_rm +' + +test_expect_success "'ipfs block rm' output looks good" ' + echo "removed $HASH" > expected_rm && + test_cmp expected_rm actual_rm +' + +test_expect_success "'ipfs block rm' block actually removed" ' + test_must_fail ipfs block stat $HASH +' + +DIRHASH=QmdWmVmM6W2abTgkEfpbtA1CJyTWS2rhuUB9uP1xV8Uwtf +FILE1HASH=Qmae3RedM7SNkWGsdzYzsr6svmsFdsva4WoTvYYsWhUSVz +FILE2HASH=QmUtkGLvPf63NwVzLPKPUYgwhn8ZYPWF6vKWN3fZ2amfJF +FILE3HASH=Qmesmmf1EEG1orJb6XdK6DabxexsseJnCfw8pqWgonbkoj + +test_expect_success "add and pin directory" ' + mkdir adir && + echo "file1" > adir/file1 && + echo "file2" > adir/file2 && + echo "file3" > adir/file3 && + ipfs add -r adir + ipfs pin add -r $DIRHASH +' + +test_expect_success "can't remove pinned block" ' + test_must_fail ipfs block rm $DIRHASH 2> block_rm_err +' + +test_expect_success "can't remove pinned block: output looks good" ' + grep -q "$DIRHASH: pinned: recursive" block_rm_err +' + +test_expect_success "can't remove indirectly pinned block" ' + test_must_fail ipfs block rm $FILE1HASH 2> block_rm_err +' + +test_expect_success "can't remove indirectly pinned block: output looks good" ' + grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err +' + +test_expect_success "remove pin" ' + ipfs pin rm -r $DIRHASH +' + +test_expect_success "multi-block 'ipfs block rm' succeeds" ' + ipfs block rm $FILE1HASH $FILE2HASH $FILE3HASH > actual_rm +' + +test_expect_success "multi-block 'ipfs block rm' output looks good" ' + grep -F -q "removed $FILE1HASH" actual_rm && + grep -F -q "removed $FILE2HASH" actual_rm && + grep -F -q "removed $FILE3HASH" actual_rm +' + +test_expect_success "'add some blocks' succeeds" ' + echo "Hello Mars!" | ipfs block put && + echo "Hello Venus!" | ipfs block put +' + +test_expect_success "add and pin directory" ' + ipfs add -r adir + ipfs pin add -r $DIRHASH +' + +HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk +HASH2=QmdnpnsaEj69isdw5sNzp3h3HkaDz7xKq7BmvFFBzNr5e7 +RANDOMHASH=QRmKqGMAM6EbngbZjSqrvYzq5Qd8b1bSWymjSUY9zQSNDq + +test_expect_success "multi-block 'ipfs block rm' mixed" ' + test_must_fail ipfs block rm $FILE1HASH $DIRHASH $HASH $FILE3HASH $RANDOMHASH $HASH2 2> block_rm_err +' + +test_expect_success "pinned block not removed" ' + ipfs block stat $FILE1HASH && + ipfs block stat $FILE3HASH +' + +test_expect_success "non-pinned blocks removed" ' + test_must_fail ipfs block stat $HASH && + test_must_fail ipfs block stat $HASH2 +' + +test_expect_success "error reported on removing non-existent block" ' + grep -q "cannot remove $RANDOMHASH" block_rm_err +' + +test_expect_success "'add some blocks' succeeds" ' + echo "Hello Mars!" | ipfs block put && + echo "Hello Venus!" | ipfs block put +' + +test_expect_success "multi-block 'ipfs block rm -f' with non existent blocks succeed" ' + ipfs block rm -f $HASH $RANDOMHASH $HASH2 +' + +test_expect_success "existent blocks removed" ' + test_must_fail ipfs block stat $HASH && + test_must_fail ipfs block stat $HASH2 +' + +test_expect_success "'add some blocks' succeeds" ' + echo "Hello Mars!" | ipfs block put && + echo "Hello Venus!" | ipfs block put +' + +test_expect_success "multi-block 'ipfs block rm -q' produces no output" ' + ipfs block rm -q $HASH $HASH2 > block_rm_out && + test ! -s block_rm_out +' + +# +# Misc tests +# + test_expect_success "'ipfs block stat' with nothing from stdin doesnt crash" ' test_expect_code 1 ipfs block stat < /dev/null 2> stat_out '