Merge branch 'v2.0.3-p4' into develop

node/consensus/data/data_clock_consensus_engine.go

@@ -479,9 +479,9 @@ func (e *DataClockConsensusEngine) Start() <-chan error {
 		time.Sleep(30 * time.Second)
 		e.logger.Info("checking for snapshots to play forward")
 		if err := e.downloadSnapshot(e.config.DB.Path, e.config.P2P.Network); err != nil {
-			e.logger.Error("error downloading snapshot", zap.Error(err))
+			e.logger.Debug("error downloading snapshot", zap.Error(err))
 		} else if err := e.applySnapshot(e.config.DB.Path); err != nil {
-			e.logger.Error("error replaying snapshot", zap.Error(err))
+			e.logger.Debug("error replaying snapshot", zap.Error(err))
 		}
 	}()
 

node/consensus/time/data_time_reel.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"math/big"
+	"os"
 	"sync"
 
 	lru "github.com/hashicorp/golang-lru/v2"
@@ -149,6 +150,16 @@ func (d *DataTimeReel) Start() error {
 		d.headDistance = big.NewInt(0)
 	} else {
 		if len(tries[0].FindNearestAndApproximateNeighbors(make([]byte, 32))) == 0 {
+			if frame.FrameNumber > 53027 {
+				d.logger.Info("DANGER")
+				d.logger.Info("DANGER")
+				d.logger.Info("DANGER")
+				d.logger.Info("It appears your node is running with a broken store. Please restore from backup or create a new store.")
+				d.logger.Info("DANGER")
+				d.logger.Info("DANGER")
+				d.logger.Info("DANGER")
+				os.Exit(1)
+			}
 			d.logger.Info("encountered trie corruption, invoking restoration")
 			tries = d.restore()
 		}

node/execution/intrinsics/token/token_execution_engine.go

@@ -659,8 +659,9 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				peer := new(big.Int).SetUint64(sen.seniority)
 				if peer.Cmp(GetAggregatedSeniority([]string{peerId})) != 0 {
 					logger(
-						"peer announced but is already different seniority",
-						zap.String("peer_id", peerIds[0]),
+						"peer announced but has already been announced",
+						zap.String("peer_id", peerId),
+						zap.Uint64("seniority", sen.seniority),
 					)
 					mergeable = false
 					break
@@ -743,8 +744,9 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				peer := new(big.Int).SetUint64(sen.seniority)
 				if peer.Cmp(GetAggregatedSeniority([]string{peerIds[0]})) != 0 {
 					logger(
-						"peer announced but is already different seniority",
+						"peer announced but has already been announced",
 						zap.String("peer_id", peerIds[0]),
+						zap.Uint64("seniority", sen.seniority),
 					)
 					continue
 				}

node/store/clock.go

@@ -1538,6 +1538,7 @@ func (p *PebbleClockStore) SetProverTriesForFrame(
 	frame *protobufs.ClockFrame,
 	tries []*tries.RollingFrecencyCritbitTrie,
 ) error {
+	start := 0
 	for i, proverTrie := range tries {
 		proverData, err := proverTrie.Serialize()
 		if err != nil {
@@ -1550,6 +1551,26 @@ func (p *PebbleClockStore) SetProverTriesForFrame(
 		); err != nil {
 			return errors.Wrap(err, "set prover tries for frame")
 		}
+		start = i
+	}
+
+	start++
+	for {
+		_, closer, err := p.db.Get(
+			clockProverTrieKey(frame.Filter, uint16(start), frame.FrameNumber),
+		)
+		if err != nil {
+			break
+		}
+		closer.Close()
+
+		if err = p.db.Delete(
+			clockProverTrieKey(frame.Filter, uint16(start), frame.FrameNumber),
+		); err != nil {
+			return errors.Wrap(err, "set prover tries for frame")
+		}
+
+		start++
 	}
 
 	return nil