From 46bcdce15d73d20b2f3a2def2136ae6b62c120c0 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 27 Apr 2016 13:28:53 -0700 Subject: [PATCH] commands: repo fsck (#2597) * Adds repo fsck subcommand Fixes #2457 License: MIT Signed-off-by: Mike Pfister * Checks for error on file deletion License: MIT Signed-off-by: Mike Pfister * Checks if node is online License: MIT Signed-off-by: Mike Pfister * Update error checking License: MIT Signed-off-by: Michael Pfister * Prevents command from running while daemon is running License: MIT Signed-off-by: Michael Pfister * Add newline to command output message License: MIT Signed-off-by: Michael Pfister * removing superfluous error License: MIT Signed-off-by: Michael Pfister * Adds sharness test for repo fsck command License: MIT Signed-off-by: Michael Pfister * Ignore warning if file doesn't exist License: MIT Signed-off-by: Michael Pfister * Updating message output License: MIT Signed-off-by: Michael Pfister * adding debug statements License: MIT Signed-off-by: Michael Pfister * update and add fsck sharness tests License: MIT Signed-off-by: Michael Pfister * updating comments License: MIT Signed-off-by: Michael Pfister * Use printf in test Using printf prevents a newline from being printed to the api test file. When the newline was present, multiaddr threw errors trying to parse the api address to an integer since the newline character was present. License: MIT Signed-off-by: Michael Pfister * updating tests License: MIT Signed-off-by: Michael Pfister * removing commented code License: MIT Signed-off-by: Michael Pfister --- cmd/ipfs/init.go | 1 + cmd/ipfs/ipfs.go | 1 + cmd/ipfs/main.go | 3 +- core/commands/repo.go | 58 +++++++++- repo/fsrepo/fsrepo.go | 9 +- repo/fsrepo/lock/lock.go | 11 ++ test/sharness/t0083-repo-fsck.sh | 190 +++++++++++++++++++++++++++++++ 7 files changed, 268 insertions(+), 5 deletions(-) create mode 100755 test/sharness/t0083-repo-fsck.sh diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index 2861901fb..ebc5f6069 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -51,6 +51,7 @@ at ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable log.Info("checking if daemon is running...") if daemonLocked { + log.Debug("Ipfs daemon is running.") e := "ipfs daemon is running. please stop it to run this command" return cmds.ClientError(e) } diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index 179d6cfb7..57b6e9091 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -105,4 +105,5 @@ var cmdDetailsMap = map[*cmds.Command]cmdDetails{ commands.CommandsDaemonCmd: {doesNotUseRepo: true}, commands.VersionCmd: {doesNotUseConfigAsInput: true, doesNotUseRepo: true}, // must be permitted to run before init commands.LogCmd: {cannotRunOnClient: true}, + commands.RepoFsckCmd: {cannotRunOnDaemon: true}, } diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index 563c83e10..40757f9df 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -417,9 +417,10 @@ func commandShouldRunOnDaemon(details cmdDetails, req cmds.Request, root *cmds.C return nil, err } - if client != nil { // api file exists + if client != nil { if details.cannotRunOnDaemon { // check if daemon locked. legacy error text, for now. + log.Debugf("Command cannot run on daemon. Checking if daemon is locked") if daemonLocked, _ := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot); daemonLocked { return nil, cmds.ClientError("ipfs daemon is running. please stop it to run this command") } diff --git a/core/commands/repo.go b/core/commands/repo.go index ff4615ef7..2aaadb67d 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -5,8 +5,12 @@ import ( "fmt" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" + config "github.com/ipfs/go-ipfs/repo/config" + lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" "io" + "os" + "path/filepath" ) var RepoCmd = &cmds.Command{ @@ -20,6 +24,7 @@ var RepoCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "gc": repoGcCmd, "stat": repoStatCmd, + "fsck": RepoFsckCmd, }, } @@ -32,7 +37,6 @@ set of stored objects and remove ones that are not pinned in order to reclaim hard disk space. `, }, - Options: []cmds.Option{ cmds.BoolOption("quiet", "q", "Write minimal output."), }, @@ -152,3 +156,55 @@ RepoSize int Size in bytes that the repo is currently taking. }, }, } + +var RepoFsckCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Removes repo lockfiles", + ShortDescription: ` +'ipfs repo fsck' is a plumbing command that will remove repo and level db +lockfiles, as well as the api file. This command can only run when no ipfs +daemons are running. +`, + }, + Run: func(req cmds.Request, res cmds.Response) { + configRoot := req.InvocContext().ConfigRoot + + dsPath, err := config.DataStorePath(configRoot) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + dsLockFile := filepath.Join(dsPath, "LOCK") // TODO: get this lockfile programmatically + repoLockFile := filepath.Join(configRoot, lockfile.LockFile) + apiFile := filepath.Join(configRoot, "api") // TODO: get this programmatically + + log.Infof("Removing repo lockfile: %s", repoLockFile) + log.Infof("Removing datastore lockfile: %s", dsLockFile) + log.Infof("Removing api file: %s", apiFile) + + err = os.Remove(repoLockFile) + if err != nil && !os.IsNotExist(err) { + res.SetError(err, cmds.ErrNormal) + return + } + err = os.Remove(dsLockFile) + if err != nil && !os.IsNotExist(err) { + res.SetError(err, cmds.ErrNormal) + return + } + err = os.Remove(apiFile) + if err != nil && !os.IsNotExist(err) { + res.SetError(err, cmds.ErrNormal) + return + } + + s := "Lockfiles have been removed." + log.Info(s) + res.SetOutput(&MessageOutput{s + "\n"}) + }, + Type: MessageOutput{}, + Marshalers: cmds.MarshalerMap{ + cmds.Text: MessageTextMarshaler, + }, +} diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 9dcfb86ff..ee86a41f3 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -259,8 +259,11 @@ func Remove(repoPath string) error { // process. If true, then the repo cannot be opened by this process. func LockedByOtherProcess(repoPath string) (bool, error) { repoPath = filepath.Clean(repoPath) - // NB: the lock is only held when repos are Open - return lockfile.Locked(repoPath) + locked, err := lockfile.Locked(repoPath) + if locked { + log.Debugf("(%t)<->Lock is held at %s", locked, repoPath) + } + return locked, err } // APIAddr returns the registered API addr, according to the api file @@ -360,7 +363,7 @@ func (r *FSRepo) Close() error { } err := os.Remove(filepath.Join(r.path, apiFile)) - if err != nil { + if err != nil && !os.IsNotExist(err) { log.Warning("error removing api file: ", err) } diff --git a/repo/fsrepo/lock/lock.go b/repo/fsrepo/lock/lock.go index 34f2d2339..f53703aff 100644 --- a/repo/fsrepo/lock/lock.go +++ b/repo/fsrepo/lock/lock.go @@ -10,12 +10,16 @@ import ( lock "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/camlistore/lock" "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" + logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log" ) // LockFile is the filename of the repo lock, relative to config dir // TODO rename repo lock and hide name const LockFile = "repo.lock" +// log is the fsrepo logger +var log = logging.Logger("lock") + func errPerm(path string) error { return fmt.Errorf("failed to take lock at %s: permission denied", path) } @@ -26,29 +30,36 @@ func Lock(confdir string) (io.Closer, error) { } func Locked(confdir string) (bool, error) { + log.Debugf("Checking lock") if !util.FileExists(path.Join(confdir, LockFile)) { + log.Debugf("File doesn't exist: %s", path.Join(confdir, LockFile)) return false, nil } if lk, err := Lock(confdir); err != nil { // EAGAIN == someone else has the lock if err == syscall.EAGAIN { + log.Debugf("Someone else has the lock: %s", path.Join(confdir, LockFile)) return true, nil } if strings.Contains(err.Error(), "can't Lock file") { + log.Debugf("Can't lock file: %s.\n reason: %s", path.Join(confdir, LockFile), err.Error()) return true, nil } // lock fails on permissions error if os.IsPermission(err) { + log.Debugf("Lock fails on permissions error") return false, errPerm(confdir) } if isLockCreatePermFail(err) { + log.Debugf("Lock fails on permissions error") return false, errPerm(confdir) } // otherwise, we cant guarantee anything, error out return false, err } else { + log.Debugf("No one has a lock") lk.Close() return false, nil } diff --git a/test/sharness/t0083-repo-fsck.sh b/test/sharness/t0083-repo-fsck.sh new file mode 100755 index 000000000..b53f846ab --- /dev/null +++ b/test/sharness/t0083-repo-fsck.sh @@ -0,0 +1,190 @@ +#!/bin/sh +# +# Copyright (c) 2016 Mike Pfister +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test ipfs repo fsck operations" + +. lib/test-lib.sh + +test_init_ipfs + +############################# +# Test without daemon running +############################# +# Note: if api file isn't present we can assume the daemon isn't running + +# Try with all lock files present: repo.lock, api, and datastore/LOCK with +# repo.lock and datastore/LOCK being empty +test_expect_success "'ipfs repo fsck' succeeds with no daemon running empty +repo.lock" ' + mkdir -p $IPFS_PATH && + mkdir -p $IPFS_PATH/datastore && + touch $IPFS_PATH/datastore/LOCK && + touch $IPFS_PATH/repo.lock && + printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api && + ipfs repo fsck > fsck_out_actual1 +' +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual1 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +# Try with all lock files present: repo.lock, api, and datastore/LOCK with +# repo.lock is non-zero TODO: this test is broken until we find consensus on the +# non-zero repo.lock issue +test_expect_success "'ipfs repo fsck' succeeds with no daemon running non-zero +repo.lock" ' + mkdir -p $IPFS_PATH && + printf ":D" > $IPFS_PATH/repo.lock && + touch $IPFS_PATH/datastore/LOCK && + ipfs repo fsck > fsck_out_actual1b +' +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual1b +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +######################## +# Test for partial locks +######################## + +# Try with locks api and datastore/LOCK +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api && + touch $IPFS_PATH/datastore/LOCK && + ipfs repo fsck > fsck_out_actual2 +' + +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual2 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +# Try with locks api and repo.lock +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api && + touch $IPFS_PATH/repo.lock && + ipfs repo fsck > fsck_out_actual3 +' + +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual3 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +# Try with locks repo.lock and datastore +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + touch $IPFS_PATH/repo.lock && + touch $IPFS_PATH/datastore/LOCK && + ipfs repo fsck > fsck_out_actual4 +' + +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual4 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +####################### +# Test for single locks +####################### + +# Try with single locks repo.lock +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + touch $IPFS_PATH/repo.lock && + ipfs repo fsck > fsck_out_actual5 +' +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual5 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +# Try with single locks datastore/LOCK +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + touch $IPFS_PATH/datastore/LOCK && + ipfs repo fsck > fsck_out_actual6 +' +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual6 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +# Try with single lock api +test_expect_success "'ipfs repo fsck' succeeds partial lock" ' + printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api && + ipfs repo fsck > fsck_out_actual7 +' + +test_expect_success "'ipfs repo fsck' output looks good with no daemon" ' + grep "Lockfiles have been removed." fsck_out_actual7 +' + +# Make sure the files are actually removed +test_expect_success "'ipfs repo fsck' confirm file deletion" ' + test ! -e "$IPFS_PATH/repo.lock" && + test ! -e "$IPFS_PATH/datastore/LOCK" && + test ! -e "$IPFS_PATH/api" +' + +########################## +# Test with daemon running +########################## + +test_launch_ipfs_daemon + +# Daemon is running -> command doesn't run +test_expect_success "'ipfs repo fsck' fails with daemon running" ' + ! (ipfs repo fsck 2>fsck_out_actual8 ) + +' + +test_expect_success "'ipfs repo fsck' output looks good with daemon" ' + grep "Error: ipfs daemon is running" fsck_out_actual8 +' + +test_kill_ipfs_daemon + +test_done