mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-27 05:17:49 +08:00
filestore util: Add 'filestore rm' command.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
This commit is contained in:
Kevin Atkinson
parent
96b87238fe
commit
32e09a6778
@ -6,6 +6,8 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
|
||||
butil "github.com/ipfs/go-ipfs/blocks/blockstore/util"
|
||||
oldCmds "github.com/ipfs/go-ipfs/commands"
|
||||
"github.com/ipfs/go-ipfs/core"
|
||||
e "github.com/ipfs/go-ipfs/core/commands/e"
|
||||
@ -22,6 +24,7 @@ var FileStoreCmd = &cmds.Command{
|
||||
},
|
||||
Subcommands: map[string]*cmds.Command{
|
||||
"ls": lsFileStore,
|
||||
"rm": rmFileStore,
|
||||
},
|
||||
OldSubcommands: map[string]*oldCmds.Command{
|
||||
"verify": verifyFileStore,
|
||||
@ -236,6 +239,63 @@ var dupsFileStore = &oldCmds.Command{
|
||||
Type: RefWrapper{},
|
||||
}
|
||||
|
||||
var rmFileStore = &cmds.Command{
|
||||
Helptext: cmdkit.HelpText{
|
||||
Tagline: "Remove IPFS block(s) from just the filestore or blockstore.",
|
||||
ShortDescription: `
|
||||
Remove blocks from either the filestore or the main blockstore.
|
||||
`,
|
||||
},
|
||||
Arguments: []cmdkit.Argument{
|
||||
cmdkit.StringArg("hash", true, true, "CID's of block(s) to remove."),
|
||||
},
|
||||
Options: []cmdkit.Option{
|
||||
cmdkit.BoolOption("force", "f", "Ignore nonexistent blocks."),
|
||||
cmdkit.BoolOption("quiet", "q", "Write minimal output."),
|
||||
cmdkit.BoolOption("non-filestore", "Remove non-filestore blocks"),
|
||||
},
|
||||
Run: func(req cmds.Request, res cmds.ResponseEmitter) {
|
||||
n, fs, err := getFilestore(req.InvocContext())
|
||||
if err != nil {
|
||||
res.SetError(err, cmdkit.ErrNormal)
|
||||
return
|
||||
}
|
||||
hashes := req.Arguments()
|
||||
force, _, _ := req.Option("force").Bool()
|
||||
quiet, _, _ := req.Option("quiet").Bool()
|
||||
nonFilestore, _, _ := req.Option("non-filestore").Bool()
|
||||
prefix := filestore.FilestorePrefix.String()
|
||||
if nonFilestore {
|
||||
prefix = bs.BlockPrefix.String()
|
||||
}
|
||||
cids := make([]*cid.Cid, 0, len(hashes))
|
||||
for _, hash := range hashes {
|
||||
c, err := cid.Decode(hash)
|
||||
if err != nil {
|
||||
res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmdkit.ErrNormal)
|
||||
return
|
||||
}
|
||||
|
||||
cids = append(cids, c)
|
||||
}
|
||||
ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{
|
||||
Prefix: prefix,
|
||||
Quiet: quiet,
|
||||
Force: force,
|
||||
})
|
||||
if err != nil {
|
||||
res.SetError(err, cmdkit.ErrNormal)
|
||||
return
|
||||
}
|
||||
err = res.Emit(ch)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
},
|
||||
PostRun: blockRmCmd.PostRun,
|
||||
Type: butil.RemovedBlock{},
|
||||
}
|
||||
|
||||
type getNoder interface {
|
||||
GetNode() (*core.IpfsNode, error)
|
||||
}
|
||||
|
||||
91
filestore/remove.go
Normal file
91
filestore/remove.go
Normal file
@ -0,0 +1,91 @@
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
|
||||
u "github.com/ipfs/go-ipfs/blocks/blockstore/util"
|
||||
"github.com/ipfs/go-ipfs/pin"
|
||||
|
||||
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
|
||||
ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
|
||||
)
|
||||
|
||||
// RmBlocks removes blocks from either the filestore or the
|
||||
// blockstore. It is similar to blockstore_util.RmBlocks but allows
|
||||
// the removal of pinned block from one store if it is also in the
|
||||
// other.
|
||||
func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) {
|
||||
// make the channel large enough to hold any result to avoid
|
||||
// blocking while holding the GCLock
|
||||
out := make(chan interface{}, len(cids))
|
||||
|
||||
var blocks deleter
|
||||
switch opts.Prefix {
|
||||
case FilestorePrefix.String():
|
||||
blocks = fs.fm
|
||||
case bs.BlockPrefix.String():
|
||||
blocks = fs.bs
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown prefix: %s", opts.Prefix)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(out)
|
||||
|
||||
unlocker := lock.GCLock()
|
||||
defer unlocker.Unlock()
|
||||
|
||||
stillOkay := filterPinned(fs, pins, out, cids, blocks)
|
||||
|
||||
for _, c := range stillOkay {
|
||||
err := blocks.DeleteBlock(c)
|
||||
if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
|
||||
// ignore non-existent blocks
|
||||
} else if err != nil {
|
||||
out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()}
|
||||
} else if !opts.Quiet {
|
||||
out <- &u.RemovedBlock{Hash: c.String()}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type deleter interface {
|
||||
DeleteBlock(c *cid.Cid) error
|
||||
}
|
||||
|
||||
func filterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn deleter) []*cid.Cid {
|
||||
stillOkay := make([]*cid.Cid, 0, len(cids))
|
||||
res, err := pins.CheckIfPinned(cids...)
|
||||
if err != nil {
|
||||
out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
|
||||
return nil
|
||||
}
|
||||
for _, r := range res {
|
||||
if !r.Pinned() || availableElsewhere(fs, foundIn, r.Key) {
|
||||
stillOkay = append(stillOkay, r.Key)
|
||||
} else {
|
||||
out <- &u.RemovedBlock{
|
||||
Hash: r.Key.String(),
|
||||
Error: r.String(),
|
||||
}
|
||||
}
|
||||
}
|
||||
return stillOkay
|
||||
}
|
||||
|
||||
func availableElsewhere(fs *Filestore, foundIn deleter, c *cid.Cid) bool {
|
||||
switch {
|
||||
case fs.fm == foundIn:
|
||||
have, _ := fs.bs.Has(c)
|
||||
return have
|
||||
case fs.bs == foundIn:
|
||||
have, _ := fs.fm.Has(c)
|
||||
return have
|
||||
default:
|
||||
// programmer error
|
||||
panic("invalid pointer for foundIn")
|
||||
}
|
||||
}
|
||||
@ -153,7 +153,7 @@ test_filestore_verify() {
|
||||
test_init_dataset
|
||||
}
|
||||
|
||||
test_filestore_dups() {
|
||||
test_filestore_dups_and_rm() {
|
||||
# make sure the filestore is in a clean state
|
||||
test_filestore_state
|
||||
|
||||
@ -163,6 +163,28 @@ test_filestore_dups() {
|
||||
echo "$FILE1_HASH" > dups_expect
|
||||
test_cmp dups_expect dups_actual
|
||||
'
|
||||
|
||||
test_expect_success "remove non-filestore block of dup ok" '
|
||||
ipfs filestore rm --non-filestore $FILE1_HASH &&
|
||||
ipfs filestore dups > dups_actual &&
|
||||
test_cmp /dev/null dups_actual
|
||||
'
|
||||
|
||||
test_expect_success "block still in filestore" '
|
||||
ipfs filestore ls $FILE1_HASH | grep -q file1
|
||||
'
|
||||
|
||||
test_expect_success "remove non-duplicate pinned block not ok" '
|
||||
test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err &&
|
||||
grep -q pinned rm_err
|
||||
'
|
||||
|
||||
test_expect_success "remove filestore block of dup ok" '
|
||||
ipfs add --raw-leaves somedir/file1 &&
|
||||
ipfs filestore rm $FILE1_HASH &&
|
||||
ipfs filestore dups > dups_actual &&
|
||||
test_cmp /dev/null dups_actual
|
||||
'
|
||||
}
|
||||
|
||||
#
|
||||
@ -175,7 +197,7 @@ test_filestore_adds
|
||||
|
||||
test_filestore_verify
|
||||
|
||||
test_filestore_dups
|
||||
test_filestore_dups_and_rm
|
||||
|
||||
#
|
||||
# With daemon
|
||||
@ -191,7 +213,7 @@ test_filestore_adds
|
||||
|
||||
test_filestore_verify
|
||||
|
||||
test_filestore_dups
|
||||
test_filestore_dups_and_rm
|
||||
|
||||
test_kill_ipfs_daemon
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user