diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go
index 1e2e34143..a2c135126 100644
--- a/p2p/net/swarm/swarm_dial.go
+++ b/p2p/net/swarm/swarm_dial.go
@@ -227,14 +227,20 @@ func (s *Swarm) gatedDialAttempt(ctx context.Context, p peer.ID) (*Conn, error)
 
 	// check if there's an ongoing dial to this peer
 	if ok, wait := s.dsync.Lock(p); ok {
+		defer s.dsync.Unlock(p)
+
+		// if this peer has been backed off, lets get out of here
+		if s.backf.Backoff(p) {
+			log.Event(ctx, "swarmDialBackoff", logdial)
+			return nil, ErrDialBackoff
+		}
+
 		// ok, we have been charged to dial! let's do it.
 		// if it succeeds, dial will add the conn to the swarm itself.
-
 		defer log.EventBegin(ctx, "swarmDialAttemptStart", logdial).Done()
 		ctxT, cancel := context.WithTimeout(ctx, s.dialT)
 		conn, err := s.dial(ctxT, p)
 		cancel()
-		s.dsync.Unlock(p)
 		log.Debugf("dial end %s", conn)
 		if err != nil {
 			log.Event(ctx, "swarmDialBackoffAdd", logdial)