From a3d3a8d795d4c93327498acd9cb0c9532e612038 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 30 Oct 2025 02:19:35 -0500 Subject: [PATCH] bulk of tests --- consensus/consensus_committee.go | 4 +- consensus/consensus_forks.go | 2 +- consensus/consensus_signature.go | 2 +- consensus/consensus_signer.go | 2 +- consensus/consensus_timeout.go | 2 +- consensus/consensus_verifier.go | 2 +- .../event_handler.go | 5 +- consensus/eventhandler/event_handler_test.go | 1120 ++++++ .../{event_loop => eventloop}/event_loop.go | 95 +- consensus/eventloop/event_loop_test.go | 250 ++ .../example/generic_consensus_example.go | 607 ++-- consensus/forest/leveled_forest.go | 4 +- consensus/forest/vertex.go | 2 +- consensus/forks/forks.go | 38 +- consensus/forks/forks_test.go | 950 ++++++ consensus/forks/state_builder_test.go | 165 + consensus/go.mod | 56 +- consensus/go.sum | 117 +- consensus/helper/quorum_certificate.go | 10 +- consensus/helper/state.go | 230 ++ consensus/helper/timeout_certificate.go | 18 + consensus/mocks/dynamic_committee.go | 10 +- consensus/mocks/signer.go | 12 +- consensus/mocks/verifying_vote_processor.go | 8 +- consensus/mocks/vote_collector.go | 36 +- consensus/mocks/vote_processor_factory.go | 18 +- consensus/mocks/voting_provider.go | 30 + consensus/models/state.go | 8 +- consensus/models/timeout_state.go | 23 +- .../pubsub/communicator_distributor.go | 2 +- .../pubsub/participant_distributor.go | 46 +- consensus/pacemaker/pacemaker.go | 1 + consensus/safetyrules/safety_rules.go | 5 +- consensus/safetyrules/safety_rules_test.go | 834 +++++ consensus/signature/state_signer_decoder.go | 2 +- .../weighted_signature_aggregator.go | 6 +- consensus/state_machine.go | 3000 +++++++++-------- .../timeout_aggregator_test.go | 133 + .../timeout_collectors_test.go | 176 + consensus/timeoutcollector/aggregation.go | 2 +- consensus/timeoutcollector/factory.go | 2 + .../timeoutcollector/timeout_cache_test.go | 172 + .../timeout_collector_test.go | 230 ++ .../timeoutcollector/timeout_processor.go | 6 +- .../timeout_processor_test.go | 678 ++++ consensus/tracker/tracker.go | 36 +- consensus/validator/validator.go | 14 +- consensus/verification/common.go | 2 +- consensus/vote_aggregator.go | 2 +- consensus/vote_collector.go | 14 +- consensus/voteaggregator/vote_aggregator.go | 4 +- consensus/voteaggregator/vote_collectors.go | 14 +- consensus/votecollector/common.go | 10 +- consensus/votecollector/factory.go | 46 +- consensus/votecollector/statemachine.go | 75 +- consensus/votecollector/vote_cache.go | 10 +- consensus/votecollector/vote_processor.go | 4 +- 57 files changed, 7366 insertions(+), 1986 deletions(-) rename consensus/{event_handler => eventhandler}/event_handler.go (99%) create mode 100644 consensus/eventhandler/event_handler_test.go rename consensus/{event_loop => eventloop}/event_loop.go (78%) create mode 100644 consensus/eventloop/event_loop_test.go create mode 100644 consensus/forks/forks_test.go create mode 100644 consensus/forks/state_builder_test.go create mode 100644 consensus/safetyrules/safety_rules_test.go create mode 100644 consensus/timeoutaggregator/timeout_aggregator_test.go create mode 100644 consensus/timeoutaggregator/timeout_collectors_test.go create mode 100644 consensus/timeoutcollector/timeout_cache_test.go create mode 100644 consensus/timeoutcollector/timeout_collector_test.go create mode 100644 consensus/timeoutcollector/timeout_processor_test.go diff --git a/consensus/consensus_committee.go b/consensus/consensus_committee.go index 85cc776..4eb0d4d 100644 --- a/consensus/consensus_committee.go +++ b/consensus/consensus_committee.go @@ -132,7 +132,7 @@ type DynamicCommittee interface { IdentityByState( stateID models.Identity, participantID models.Identity, - ) (*models.WeightedIdentity, error) + ) (models.WeightedIdentity, error) } // StateSignerDecoder defines how to convert the ParentSignerIndices field @@ -146,7 +146,7 @@ type StateSignerDecoder[StateT models.Unique] interface { // parent state. Consequently, the returned IdentifierList contains the // consensus participants that signed the parent state. // Expected Error returns during normal operations: - // - signature.InvalidSignerIndicesError if signer indices included in the + // - consensus.InvalidSignerIndicesError if signer indices included in the // header do not encode a valid subset of the consensus committee DecodeSignerIDs( state *models.State[StateT], diff --git a/consensus/consensus_forks.go b/consensus/consensus_forks.go index 4c09cb7..32e0475 100644 --- a/consensus/consensus_forks.go +++ b/consensus/consensus_forks.go @@ -28,7 +28,7 @@ type Forks[StateT models.Unique] interface { // GetStatesForRank returns all known states for the given rank GetStatesForRank(rank uint64) []*models.State[StateT] - // GetState returns (*model.State, true) if the state with the specified + // GetState returns (*models.State[*helper.TestState], true) if the state with the specified // id was found and (nil, false) otherwise. GetState(stateID models.Identity) (*models.State[StateT], bool) diff --git a/consensus/consensus_signature.go b/consensus/consensus_signature.go index d0d168f..9f65891 100644 --- a/consensus/consensus_signature.go +++ b/consensus/consensus_signature.go @@ -36,7 +36,7 @@ type WeightedSignatureAggregator interface { // TotalWeight returns the total weight presented by the collected signatures. TotalWeight() uint64 - // Aggregate aggregates the signatures and returns the aggregated signature. + // Aggregate aggregates the signatures and returns the aggregated consensus. // The function performs a final verification and errors if the aggregated // signature is invalid. This is required for the function safety since // `TrustedAdd` allows adding invalid signatures. diff --git a/consensus/consensus_signer.go b/consensus/consensus_signer.go index 31b128e..a54f0ac 100644 --- a/consensus/consensus_signer.go +++ b/consensus/consensus_signer.go @@ -13,7 +13,7 @@ type Signer[StateT models.Unique, VoteT models.Unique] interface { // CreateTimeout creates a timeout for given rank. No errors return are // expected during normal operations(incl presence of byz. actors). CreateTimeout( - curView uint64, + curRank uint64, newestQC models.QuorumCertificate, previousRankTimeoutCert models.TimeoutCertificate, ) (*models.TimeoutState[VoteT], error) diff --git a/consensus/consensus_timeout.go b/consensus/consensus_timeout.go index 11cfb36..cf61580 100644 --- a/consensus/consensus_timeout.go +++ b/consensus/consensus_timeout.go @@ -53,7 +53,7 @@ type TimeoutCollector[VoteT models.Unique] interface { // TimeoutProcessor ingests Timeout States for a particular rank. It // implements the algorithms for validating TSs, orchestrates their low-level -// aggregation and emits `OnPartialTcCreated` and `OnTcConstructedFromTimeouts` +// aggregation and emits `OnPartialTimeoutCertificateCreated` and `OnTimeoutCertificateConstructedFromTimeouts` // notifications. TimeoutProcessor cannot deduplicate TSs (this should be // handled by the higher-level TimeoutCollector) and errors instead. Depending // on their implementation, a TimeoutProcessor might drop timeouts or attempt to diff --git a/consensus/consensus_verifier.go b/consensus/consensus_verifier.go index 3ff3992..c5ff1dd 100644 --- a/consensus/consensus_verifier.go +++ b/consensus/consensus_verifier.go @@ -36,7 +36,7 @@ type Verifier[VoteT models.Unique] interface { // Return values: // * nil if `sigData` is cryptographically valid // * models.InsufficientSignaturesError if `signers is empty. - // * models.InvalidFormatError if `signers`/`highQCViews` have differing + // * models.InvalidFormatError if `signers`/`highQCRanks` have differing // lengths // * models.ErrInvalidSignature if a signature is invalid // * unexpected errors should be treated as symptoms of bugs or uncovered diff --git a/consensus/event_handler/event_handler.go b/consensus/eventhandler/event_handler.go similarity index 99% rename from consensus/event_handler/event_handler.go rename to consensus/eventhandler/event_handler.go index 213bdb7..b3d2261 100644 --- a/consensus/event_handler/event_handler.go +++ b/consensus/eventhandler/event_handler.go @@ -256,7 +256,7 @@ func (e *EventHandler[ return nil } -// OnPartialTcCreated handles notification produces by the internal timeout +// OnPartialTimeoutCertificateCreated handles notification produces by the internal timeout // aggregator. If the notification is for the current rank, a corresponding // models.TimeoutState is broadcast to the consensus committee. No errors are // expected during normal operation. @@ -426,9 +426,12 @@ func (e *EventHandler[ // check that I am the primary for this rank if e.committee.Self() != currentLeader { + e.tracer.Trace("not primary") return nil } + e.tracer.Trace("primary") + // attempt to generate proposal: newestQC := e.paceMaker.LatestQuorumCertificate() previousRankTimeoutCert := e.paceMaker.PriorRankTimeoutCertificate() diff --git a/consensus/eventhandler/event_handler_test.go b/consensus/eventhandler/event_handler_test.go new file mode 100644 index 0000000..8b8dd33 --- /dev/null +++ b/consensus/eventhandler/event_handler_test.go @@ -0,0 +1,1120 @@ +package eventhandler + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" + "source.quilibrium.com/quilibrium/monorepo/consensus/pacemaker" +) + +const ( + minRepTimeout float64 = 100.0 // Milliseconds + maxRepTimeout float64 = 600.0 // Milliseconds + multiplicativeIncrease float64 = 1.5 // multiplicative factor + happyPathMaxRoundFailures uint64 = 6 // number of failed rounds before first timeout increase +) + +// TestPacemaker is a real pacemaker module with logging for rank changes +type TestPacemaker[ + StateT models.Unique, + VoteT models.Unique, + PeerIDT models.Unique, + CollectedT models.Unique, +] struct { + consensus.Pacemaker +} + +var _ consensus.Pacemaker = (*TestPacemaker[*nilUnique, *nilUnique, *nilUnique, *nilUnique])(nil) + +func NewTestPacemaker[ + StateT models.Unique, + VoteT models.Unique, + PeerIDT models.Unique, + CollectedT models.Unique, +]( + initialParameters func() *models.LivenessState, + proposalDurationProvider consensus.ProposalDurationProvider, + notifier consensus.Consumer[StateT, VoteT], + store consensus.ConsensusStore[VoteT], + traceLogger consensus.TraceLogger, +) *TestPacemaker[StateT, VoteT, PeerIDT, CollectedT] { + p, err := pacemaker.NewPacemaker[StateT, VoteT, PeerIDT, CollectedT]( + initialParameters, + proposalDurationProvider, + notifier, + store, + traceLogger, + ) + if err != nil { + panic(err) + } + return &TestPacemaker[StateT, VoteT, PeerIDT, CollectedT]{p} +} + +func (p *TestPacemaker[ + StateT, + VoteT, + PeerIDT, + CollectedT, +]) ReceiveQuorumCertificate(qc models.QuorumCertificate) (*models.NextRank, error) { + oldRank := p.CurrentRank() + newRank, err := p.Pacemaker.ReceiveQuorumCertificate(qc) + fmt.Printf("pacemaker.ReceiveQuorumCertificate old rank: %v, new rank: %v\n", oldRank, p.CurrentRank()) + return newRank, err +} + +func (p *TestPacemaker[ + StateT, + VoteT, + PeerIDT, + CollectedT, +]) ReceiveTimeoutCertificate(tc models.TimeoutCertificate) (*models.NextRank, error) { + oldRank := p.CurrentRank() + newRank, err := p.Pacemaker.ReceiveTimeoutCertificate(tc) + fmt.Printf("pacemaker.ReceiveTimeoutCertificate old rank: %v, new rank: %v\n", oldRank, p.CurrentRank()) + return newRank, err +} + +func (p *TestPacemaker[ + StateT, + VoteT, + PeerIDT, + CollectedT, +]) LatestQuorumCertificate() models.QuorumCertificate { + return p.Pacemaker.LatestQuorumCertificate() +} + +func (p *TestPacemaker[ + StateT, + VoteT, + PeerIDT, + CollectedT, +]) PriorRankTimeoutCertificate() models.TimeoutCertificate { + return p.Pacemaker.PriorRankTimeoutCertificate() +} + +type nodelay struct{} + +// TargetPublicationTime implements consensus.ProposalDurationProvider. +func (n *nodelay) TargetPublicationTime(proposalRank uint64, timeRankEntered time.Time, parentStateId models.Identity) time.Time { + return timeRankEntered +} + +var _ consensus.ProposalDurationProvider = (*nodelay)(nil) + +// using a real pacemaker for testing event handler +func initPacemaker(t require.TestingT, ctx context.Context, livenessData *models.LivenessState) consensus.Pacemaker { + notifier := &mocks.Consumer[*helper.TestState, *helper.TestVote]{} + persist := &mocks.ConsensusStore[*helper.TestVote]{} + persist.On("PutLivenessState", mock.Anything).Return(nil).Maybe() + persist.On("GetLivenessState").Return(livenessData, nil).Once() + pm := NewTestPacemaker[*helper.TestState, *helper.TestVote, *helper.TestPeer, *helper.TestCollected]( + func() *models.LivenessState { + return &models.LivenessState{ + Filter: nil, + CurrentRank: 0, + LatestQuorumCertificate: &helper.TestQuorumCertificate{}, + PriorRankTimeoutCertificate: nil, + } + }, + &nodelay{}, + notifier, + persist, + helper.Logger(), + ) + notifier.On("OnStartingTimeout", mock.Anything, mock.Anything).Return() + notifier.On("OnQuorumCertificateTriggeredRankChange", mock.Anything, mock.Anything, mock.Anything).Return() + notifier.On("OnTimeoutCertificateTriggeredRankChange", mock.Anything, mock.Anything, mock.Anything).Return() + notifier.On("OnRankChange", mock.Anything, mock.Anything).Maybe() + pm.Start(ctx) + return pm +} + +// Committee mocks hotstuff.DynamicCommittee and allows to easily control leader for some rank. +type Committee struct { + *mocks.Replicas + // to mock I'm the leader of a certain rank, add the rank into the keys of leaders field + leaders map[uint64]struct{} +} + +func NewCommittee(t *testing.T) *Committee { + committee := &Committee{ + Replicas: mocks.NewReplicas(t), + leaders: make(map[uint64]struct{}), + } + committee.On("LeaderForRank", mock.Anything).Return(func(rank uint64) models.Identity { + _, isLeader := committee.leaders[rank] + if isLeader { + return "1" + } + return "0" + }, func(rank uint64) error { + return nil + }).Maybe() + + committee.On("Self").Return("1").Maybe() + + return committee +} + +// The SafetyRules mock will not vote for any state unless the state's ID exists in votable field's key +type SafetyRules struct { + *mocks.SafetyRules[*helper.TestState, *helper.TestVote] + votable map[models.Identity]struct{} +} + +func NewSafetyRules(t *testing.T) *SafetyRules { + safetyRules := &SafetyRules{ + SafetyRules: mocks.NewSafetyRules[*helper.TestState, *helper.TestVote](t), + votable: make(map[models.Identity]struct{}), + } + + // SafetyRules will not vote for any state, unless the stateID exists in votable map + safetyRules.On("ProduceVote", mock.Anything, mock.Anything).Return( + func(state *models.SignedProposal[*helper.TestState, *helper.TestVote], _ uint64) **helper.TestVote { + _, ok := safetyRules.votable[state.State.Identifier] + if !ok { + return nil + } + v := createVote(state.State) + return &v + }, + func(state *models.SignedProposal[*helper.TestState, *helper.TestVote], _ uint64) error { + _, ok := safetyRules.votable[state.State.Identifier] + if !ok { + return models.NewNoVoteErrorf("state not found") + } + return nil + }).Maybe() + + safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return( + func(curRank uint64, newestQC models.QuorumCertificate, lastRankTC models.TimeoutCertificate) *models.TimeoutState[*helper.TestVote] { + return helper.TimeoutStateFixture(func(timeout *models.TimeoutState[*helper.TestVote]) { + timeout.Rank = curRank + timeout.LatestQuorumCertificate = newestQC + timeout.PriorRankTimeoutCertificate = lastRankTC + }) + }, + func(uint64, models.QuorumCertificate, models.TimeoutCertificate) error { return nil }).Maybe() + + return safetyRules +} + +// Forks mock allows to customize the AddState function by specifying the addProposal callbacks +type Forks struct { + *mocks.Forks[*helper.TestState] + // proposals stores all the proposals that have been added to the forks + proposals map[models.Identity]*models.State[*helper.TestState] + finalized uint64 + t require.TestingT + // addProposal is to customize the logic to change finalized rank + addProposal func(state *models.State[*helper.TestState]) error +} + +func NewForks(t *testing.T, finalized uint64) *Forks { + f := &Forks{ + Forks: mocks.NewForks[*helper.TestState](t), + proposals: make(map[models.Identity]*models.State[*helper.TestState]), + finalized: finalized, + } + + f.On("AddValidatedState", mock.Anything).Return(func(proposal *models.State[*helper.TestState]) error { + fmt.Printf("forks.AddValidatedState received State proposal for rank: %v, QC: %v\n", proposal.Rank, proposal.ParentQuorumCertificate.GetRank()) + return f.addProposal(proposal) + }).Maybe() + + f.On("FinalizedRank").Return(func() uint64 { + return f.finalized + }).Maybe() + + f.On("GetState", mock.Anything).Return(func(stateID models.Identity) *models.State[*helper.TestState] { + b := f.proposals[stateID] + return b + }, func(stateID models.Identity) bool { + b, ok := f.proposals[stateID] + var rank uint64 + if ok { + rank = b.Rank + } + fmt.Printf("forks.GetState found %v: rank: %v\n", ok, rank) + return ok + }).Maybe() + + f.On("GetStatesForRank", mock.Anything).Return(func(rank uint64) []*models.State[*helper.TestState] { + proposals := make([]*models.State[*helper.TestState], 0) + for _, b := range f.proposals { + if b.Rank == rank { + proposals = append(proposals, b) + } + } + fmt.Printf("forks.GetStatesForRank found %v state(s) for rank %v\n", len(proposals), rank) + return proposals + }).Maybe() + + f.addProposal = func(state *models.State[*helper.TestState]) error { + f.proposals[state.Identifier] = state + if state.ParentQuorumCertificate == nil { + panic(fmt.Sprintf("state has no QC: %v", state.Rank)) + } + return nil + } + + return f +} + +// StateProducer mock will always make a valid state, exactly once per rank. +// If it is requested to make a state twice for the same rank, returns models.NoVoteError +type StateProducer struct { + proposerID models.Identity + producedStateForRank map[uint64]bool +} + +func NewStateProducer(proposerID models.Identity) *StateProducer { + return &StateProducer{ + proposerID: proposerID, + producedStateForRank: make(map[uint64]bool), + } +} + +func (b *StateProducer) MakeStateProposal(rank uint64, qc models.QuorumCertificate, lastRankTC models.TimeoutCertificate) (*models.SignedProposal[*helper.TestState, *helper.TestVote], error) { + if b.producedStateForRank[rank] { + return nil, models.NewNoVoteErrorf("state already produced") + } + b.producedStateForRank[rank] = true + return helper.MakeSignedProposal[*helper.TestState, *helper.TestVote]( + helper.WithProposal[*helper.TestState, *helper.TestVote]( + helper.MakeProposal(helper.WithState(helper.MakeState( + helper.WithStateRank[*helper.TestState](rank), + helper.WithStateQC[*helper.TestState](qc), + helper.WithStateProposer[*helper.TestState](b.proposerID))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState](lastRankTC)))), nil +} + +func TestEventHandler(t *testing.T) { + suite.Run(t, new(EventHandlerSuite)) +} + +// EventHandlerSuite contains mocked state for testing event handler under different scenarios. +type EventHandlerSuite struct { + suite.Suite + + eventhandler *EventHandler[*helper.TestState, *helper.TestVote, *helper.TestPeer, *helper.TestCollected] + + paceMaker consensus.Pacemaker + forks *Forks + persist *mocks.ConsensusStore[*helper.TestVote] + stateProducer *StateProducer + committee *Committee + notifier *mocks.Consumer[*helper.TestState, *helper.TestVote] + safetyRules *SafetyRules + + initRank uint64 // the current rank at the beginning of the test case + endRank uint64 // the expected current rank at the end of the test case + parentProposal *models.SignedProposal[*helper.TestState, *helper.TestVote] + votingProposal *models.SignedProposal[*helper.TestState, *helper.TestVote] + qc models.QuorumCertificate + tc models.TimeoutCertificate + newrank *models.NextRank + ctx context.Context + stop context.CancelFunc +} + +func (es *EventHandlerSuite) SetupTest() { + finalized := uint64(3) + + es.parentProposal = createProposal(4, 3) + newestQC := createQC(es.parentProposal.State) + + livenessData := &models.LivenessState{ + CurrentRank: newestQC.GetRank() + 1, + LatestQuorumCertificate: newestQC, + } + + es.ctx, es.stop = context.WithCancel(context.Background()) + + es.committee = NewCommittee(es.T()) + es.paceMaker = initPacemaker(es.T(), es.ctx, livenessData) + es.forks = NewForks(es.T(), finalized) + es.persist = mocks.NewConsensusStore[*helper.TestVote](es.T()) + es.persist.On("PutStarted", mock.Anything).Return(nil).Maybe() + es.stateProducer = NewStateProducer(es.committee.Self()) + es.safetyRules = NewSafetyRules(es.T()) + es.notifier = mocks.NewConsumer[*helper.TestState, *helper.TestVote](es.T()) + es.notifier.On("OnEventProcessed").Maybe() + es.notifier.On("OnEnteringRank", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnStart", mock.Anything).Maybe() + es.notifier.On("OnReceiveProposal", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnReceiveQuorumCertificate", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnReceiveTimeoutCertificate", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnPartialTimeoutCertificate", mock.Anything, mock.Anything).Maybe() + es.notifier.On("OnLocalTimeout", mock.Anything).Maybe() + es.notifier.On("OnCurrentRankDetails", mock.Anything, mock.Anything, mock.Anything).Maybe() + + eventhandler, err := NewEventHandler[*helper.TestState, *helper.TestVote, *helper.TestPeer, *helper.TestCollected]( + es.paceMaker, + es.stateProducer, + es.forks, + es.persist, + es.committee, + es.safetyRules, + es.notifier, + helper.Logger(), + ) + require.NoError(es.T(), err) + + es.eventhandler = eventhandler + + es.initRank = livenessData.CurrentRank + es.endRank = livenessData.CurrentRank + // voting state is a state for the current rank, which will trigger rank change + es.votingProposal = createProposal(es.paceMaker.CurrentRank(), es.parentProposal.State.Rank) + es.qc = helper.MakeQC(helper.WithQCState[*helper.TestState](es.votingProposal.State)) + + // create a TC that will trigger rank change for current rank, based on newest QC + es.tc = helper.MakeTC(helper.WithTCRank(es.paceMaker.CurrentRank()), + helper.WithTCNewestQC(es.votingProposal.State.ParentQuorumCertificate)) + es.newrank = &models.NextRank{ + Rank: es.votingProposal.State.Rank + 1, // the vote for the voting proposals will trigger a rank change to the next rank + } + + // add es.parentProposal into forks, otherwise we won't vote or propose based on it's QC sicne the parent is unknown + es.forks.proposals[es.parentProposal.State.Identifier] = es.parentProposal.State +} + +// TestStartNewRank_ParentProposalNotFound tests next scenario: constructed TC, it contains NewestQC that references state that we +// don't know about, proposal can't be generated because we can't be sure that resulting state payload is valid. +func (es *EventHandlerSuite) TestStartNewRank_ParentProposalNotFound() { + newestQC := helper.MakeQC(helper.WithQCRank(es.initRank + 10)) + tc := helper.MakeTC(helper.WithTCRank(newestQC.GetRank()+1), + helper.WithTCNewestQC(newestQC)) + + es.endRank = tc.GetRank() + 1 + + // I'm leader for next state + es.committee.leaders[es.endRank] = struct{}{} + + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "GetState", newestQC.GetSelector()) + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) +} + +// TestOnReceiveProposal_StaleProposal test that proposals lower than finalized rank are not processed at all +// we are not interested in this data because we already performed finalization of that height. +func (es *EventHandlerSuite) TestOnReceiveProposal_StaleProposal() { + proposal := createProposal(es.forks.FinalizedRank()-1, es.forks.FinalizedRank()-2) + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + es.forks.AssertNotCalled(es.T(), "AddState", proposal) +} + +// TestOnReceiveProposal_QCOlderThanCurrentRank tests scenario: received a valid proposal with QC that has older rank, +// the proposal's QC shouldn't trigger rank change. +func (es *EventHandlerSuite) TestOnReceiveProposal_QCOlderThanCurrentRank() { + proposal := createProposal(es.initRank-1, es.initRank-2) + + // should not trigger rank change + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "AddValidatedState", proposal.State) +} + +// TestOnReceiveProposal_TCOlderThanCurrentRank tests scenario: received a valid proposal with QC and TC that has older rank, +// the proposal's QC shouldn't trigger rank change. +func (es *EventHandlerSuite) TestOnReceiveProposal_TCOlderThanCurrentRank() { + proposal := createProposal(es.initRank-1, es.initRank-3) + proposal.PreviousRankTimeoutCertificate = helper.MakeTC(helper.WithTCRank(proposal.State.Rank-1), helper.WithTCNewestQC(proposal.State.ParentQuorumCertificate)) + + // should not trigger rank change + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "AddValidatedState", proposal.State) +} + +// TestOnReceiveProposal_NoVote tests scenario: received a valid proposal for cur rank, but not a safe node to vote, and I'm the next leader +// should not vote. +func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote() { + proposal := createProposal(es.initRank, es.initRank-1) + + // I'm the next leader + es.committee.leaders[es.initRank+1] = struct{}{} + // no vote for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "AddValidatedState", proposal.State) +} + +// TestOnReceiveProposal_NoVote_ParentProposalNotFound tests scenario: received a valid proposal for cur rank, no parent for this proposal found +// should not vote. +func (es *EventHandlerSuite) TestOnReceiveProposal_NoVote_ParentProposalNotFound() { + proposal := createProposal(es.initRank, es.initRank-1) + + // remove parent from known proposals + delete(es.forks.proposals, proposal.State.ParentQuorumCertificate.GetSelector()) + + // no vote for this proposal, no parent found + err := es.eventhandler.OnReceiveProposal(proposal) + require.Error(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "AddValidatedState", proposal.State) +} + +// TestOnReceiveProposal_Vote_NextLeader tests scenario: received a valid proposal for cur rank, safe to vote, I'm the next leader +// should vote and add vote to VoteAggregator. +func (es *EventHandlerSuite) TestOnReceiveProposal_Vote_NextLeader() { + proposal := createProposal(es.initRank, es.initRank-1) + + // I'm the next leader + es.committee.leaders[es.initRank+1] = struct{}{} + + // proposal is safe to vote + es.safetyRules.votable[proposal.State.Identifier] = struct{}{} + + vote := &helper.TestVote{ + StateID: proposal.State.Identifier, + Rank: proposal.State.Rank, + } + + es.notifier.On("OnOwnVote", mock.MatchedBy(func(v **helper.TestVote) bool { return vote.Rank == (*v).Rank && vote.StateID == (*v).StateID }), mock.Anything).Once() + + // vote should be created for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnReceiveProposal_Vote_NotNextLeader tests scenario: received a valid proposal for cur rank, safe to vote, I'm not the next leader +// should vote and send vote to next leader. +func (es *EventHandlerSuite) TestOnReceiveProposal_Vote_NotNextLeader() { + proposal := createProposal(es.initRank, es.initRank-1) + + // proposal is safe to vote + es.safetyRules.votable[proposal.State.Identifier] = struct{}{} + + vote := &helper.TestVote{ + StateID: proposal.State.Identifier, + Rank: proposal.State.Rank, + ID: "0", + } + + es.notifier.On("OnOwnVote", mock.MatchedBy(func(v **helper.TestVote) bool { + return vote.Rank == (*v).Rank && vote.StateID == (*v).StateID && vote.ID == (*v).ID + }), mock.Anything).Once() + + // vote should be created for this proposal + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnReceiveProposal_ProposeAfterReceivingTC tests a scenario where we have received TC which advances to rank where we are +// leader but no proposal can be created because we don't have parent proposal. After receiving missing parent proposal we have +// all available data to construct a valid proposal. We need to ensure this. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposeAfterReceivingQC() { + + qc := es.qc + + // first process QC this should advance rank + err := es.eventhandler.OnReceiveQuorumCertificate(qc) + require.NoError(es.T(), err) + require.Equal(es.T(), qc.GetRank()+1, es.paceMaker.CurrentRank(), "expect a rank change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) + + // we are leader for current rank + es.committee.leaders[es.paceMaker.CurrentRank()] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + proposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a header as the same as current rank + require.Equal(es.T(), es.paceMaker.CurrentRank(), proposal.State.Rank) + }).Once() + + // processing this proposal shouldn't trigger rank change since we have already seen QC. + // we have used QC to advance rounds, but no proposal was made because we were missing parent state + // when we have received parent state we can try proposing again. + err = es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + require.Equal(es.T(), qc.GetRank()+1, es.paceMaker.CurrentRank(), "expect a rank change") +} + +// TestOnReceiveProposal_ProposeAfterReceivingTC tests a scenario where we have received TC which advances to rank where we are +// leader but no proposal can be created because we don't have parent proposal. After receiving missing parent proposal we have +// all available data to construct a valid proposal. We need to ensure this. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposeAfterReceivingTC() { + + // TC contains a QC.StateID == es.votingProposal + tc := helper.MakeTC(helper.WithTCRank(es.votingProposal.State.Rank+1), + helper.WithTCNewestQC(es.qc)) + + // first process TC this should advance rank + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + require.NoError(es.T(), err) + require.Equal(es.T(), tc.GetRank()+1, es.paceMaker.CurrentRank(), "expect a rank change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) + + // we are leader for current rank + es.committee.leaders[es.paceMaker.CurrentRank()] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + proposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a header as the same as current rank + require.Equal(es.T(), es.paceMaker.CurrentRank(), proposal.State.Rank) + }).Once() + + // processing this proposal shouldn't trigger rank change, since we have already seen QC. + // we have used QC to advance rounds, but no proposal was made because we were missing parent state + // when we have received parent state we can try proposing again. + err = es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + require.Equal(es.T(), tc.GetRank()+1, es.paceMaker.CurrentRank(), "expect a rank change") +} + +// TestOnReceiveQuorumCertificate_HappyPath tests that building a QC for current rank triggers rank change. We are not leader for next +// round, so no proposal is expected. +func (es *EventHandlerSuite) TestOnReceiveQuorumCertificate_HappyPath() { + // voting state exists + es.forks.proposals[es.votingProposal.State.Identifier] = es.votingProposal.State + + // a qc is built + qc := createQC(es.votingProposal.State) + + // new qc is added to forks + // rank changed + // I'm not the next leader + // haven't received state for next rank + // goes to the new rank + es.endRank++ + // not the leader of the newrank + // don't have state for the newrank + + err := es.eventhandler.OnReceiveQuorumCertificate(qc) + require.NoError(es.T(), err, "if a vote can trigger a QC to be built,"+ + "and the QC triggered a rank change, then start new rank") + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) +} + +// TestOnReceiveQuorumCertificate_FutureRank tests that building a QC for future rank triggers rank change +func (es *EventHandlerSuite) TestOnReceiveQuorumCertificate_FutureRank() { + // voting state exists + curRank := es.paceMaker.CurrentRank() + + // b1 is for current rank + // b2 and b3 is for future rank, but branched out from the same parent as b1 + b1 := createProposal(curRank, curRank-1) + b2 := createProposal(curRank+1, curRank-1) + b3 := createProposal(curRank+2, curRank-1) + + // a qc is built + // qc3 is for future rank + // qc2 is an older than qc3 + // since vote aggregator can concurrently process votes and build qcs, + // we prepare qcs at different rank to be processed, and verify the rank change. + qc1 := createQC(b1.State) + qc2 := createQC(b2.State) + qc3 := createQC(b3.State) + + // all three proposals are known + es.forks.proposals[b1.State.Identifier] = b1.State + es.forks.proposals[b2.State.Identifier] = b2.State + es.forks.proposals[b3.State.Identifier] = b3.State + + // test that qc for future rank should trigger rank change + err := es.eventhandler.OnReceiveQuorumCertificate(qc3) + endRank := b3.State.Rank + 1 // next rank + require.NoError(es.T(), err, "if a vote can trigger a QC to be built,"+ + "and the QC triggered a rank change, then start new rank") + require.Equal(es.T(), endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + + // the same qc would not trigger rank change + err = es.eventhandler.OnReceiveQuorumCertificate(qc3) + endRank = b3.State.Rank + 1 // next rank + require.NoError(es.T(), err, "same qc should not trigger rank change") + require.Equal(es.T(), endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + + // old QCs won't trigger rank change + err = es.eventhandler.OnReceiveQuorumCertificate(qc2) + require.NoError(es.T(), err) + require.Equal(es.T(), endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + + err = es.eventhandler.OnReceiveQuorumCertificate(qc1) + require.NoError(es.T(), err) + require.Equal(es.T(), endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnReceiveQuorumCertificate_NextLeaderProposes tests that after receiving a valid proposal for cur rank, and I'm the next leader, +// a QC can be built for the state, triggered rank change, and I will propose +func (es *EventHandlerSuite) TestOnReceiveQuorumCertificate_NextLeaderProposes() { + proposal := createProposal(es.initRank, es.initRank-1) + qc := createQC(proposal.State) + // I'm the next leader + es.committee.leaders[es.initRank+1] = struct{}{} + // qc triggered rank change + es.endRank++ + // I'm the leader of cur rank (7) + // I'm not the leader of next rank (8), trigger rank change + + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + proposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a header as the same as endRank + require.Equal(es.T(), es.endRank, proposal.State.Rank) + }).Once() + + // after receiving proposal build QC and deliver it to event handler + err = es.eventhandler.OnReceiveQuorumCertificate(qc) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.forks.AssertCalled(es.T(), "AddValidatedState", proposal.State) +} + +// TestOnReceiveQuorumCertificate_ProposeOnce tests that after constructing proposal we don't attempt to create another +// proposal for same rank. +func (es *EventHandlerSuite) TestOnReceiveQuorumCertificate_ProposeOnce() { + // I'm the next leader + es.committee.leaders[es.initRank+1] = struct{}{} + + es.endRank++ + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Once() + + err := es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + // constructing QC triggers making state proposal + err = es.eventhandler.OnReceiveQuorumCertificate(es.qc) + require.NoError(es.T(), err) + + // receiving same proposal again triggers proposing logic + err = es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) +} + +// TestOnTCConstructed_HappyPath tests that building a TC for current rank triggers rank change +func (es *EventHandlerSuite) TestOnReceiveTimeoutCertificate_HappyPath() { + // voting state exists + es.forks.proposals[es.votingProposal.State.Identifier] = es.votingProposal.State + + // a tc is built + tc := helper.MakeTC(helper.WithTCRank(es.initRank), helper.WithTCNewestQC(es.votingProposal.State.ParentQuorumCertificate)) + + // expect a rank change + es.endRank++ + + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + require.NoError(es.T(), err, "TC should trigger a rank change and start of new rank") + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnTCConstructed_NextLeaderProposes tests that after receiving TC and advancing rank we as next leader create a proposal +// and broadcast it +func (es *EventHandlerSuite) TestOnReceiveTimeoutCertificate_NextLeaderProposes() { + es.committee.leaders[es.tc.GetRank()+1] = struct{}{} + es.endRank++ + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + proposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a header as the same as endRank + require.Equal(es.T(), es.endRank, proposal.State.Rank) + + // proposed state should contain valid newest QC and lastRankTC + expectedNewestQC := es.paceMaker.LatestQuorumCertificate() + require.Equal(es.T(), expectedNewestQC, proposal.State.ParentQuorumCertificate) + require.Equal(es.T(), es.paceMaker.PriorRankTimeoutCertificate(), proposal.PreviousRankTimeoutCertificate) + }).Once() + + err := es.eventhandler.OnReceiveTimeoutCertificate(es.tc) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "TC didn't trigger rank change") +} + +// TestOnTimeout tests that event handler produces TimeoutState and broadcasts it to other members of consensus +// committee. Additionally, It has to contribute TimeoutState to timeout aggregation process by sending it to TimeoutAggregator. +func (es *EventHandlerSuite) TestOnTimeout() { + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutState, ok := args[0].(*models.TimeoutState[*helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a TO with same rank as endRank + require.Equal(es.T(), es.endRank, timeoutState.Rank) + }).Once() + + err := es.eventhandler.OnLocalTimeout() + require.NoError(es.T(), err) + + // TimeoutState shouldn't trigger rank change + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnTimeout_SanityChecks tests a specific scenario where pacemaker have seen both QC and TC for previous rank +// and EventHandler tries to produce a timeout state, such timeout state is invalid if both QC and TC is present, we +// need to make sure that EventHandler filters out TC for last rank if we know about QC for same rank. +func (es *EventHandlerSuite) TestOnTimeout_SanityChecks() { + // voting state exists + es.forks.proposals[es.votingProposal.State.Identifier] = es.votingProposal.State + + // a tc is built + tc := helper.MakeTC(helper.WithTCRank(es.initRank), helper.WithTCNewestQC(es.votingProposal.State.ParentQuorumCertificate)) + + // expect a rank change + es.endRank++ + + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + require.NoError(es.T(), err, "TC should trigger a rank change and start of new rank") + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + + // receive a QC for the same rank as the TC + qc := helper.MakeQC(helper.WithQCRank(tc.GetRank())) + err = es.eventhandler.OnReceiveQuorumCertificate(qc) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "QC shouldn't trigger rank change") + require.Equal(es.T(), tc, es.paceMaker.PriorRankTimeoutCertificate(), "invalid last rank TC") + require.Equal(es.T(), qc, es.paceMaker.LatestQuorumCertificate(), "invalid newest QC") + + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutState, ok := args[0].(*models.TimeoutState[*helper.TestVote]) + require.True(es.T(), ok) + require.Equal(es.T(), es.endRank, timeoutState.Rank) + require.Equal(es.T(), qc, timeoutState.LatestQuorumCertificate) + require.Nil(es.T(), timeoutState.PriorRankTimeoutCertificate) + }).Once() + + err = es.eventhandler.OnLocalTimeout() + require.NoError(es.T(), err) +} + +// TestOnTimeout_ReplicaEjected tests that EventHandler correctly handles possible errors from SafetyRules and doesn't broadcast +// timeout states when replica is ejected. +func (es *EventHandlerSuite) TestOnTimeout_ReplicaEjected() { + es.Run("no-timeout", func() { + *es.safetyRules.SafetyRules = *mocks.NewSafetyRules[*helper.TestState, *helper.TestVote](es.T()) + es.safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return(nil, models.NewNoTimeoutErrorf("")) + err := es.eventhandler.OnLocalTimeout() + require.NoError(es.T(), err, "should be handled as sentinel error") + }) + es.Run("create-timeout-exception", func() { + *es.safetyRules.SafetyRules = *mocks.NewSafetyRules[*helper.TestState, *helper.TestVote](es.T()) + exception := errors.New("produce-timeout-exception") + es.safetyRules.On("ProduceTimeout", mock.Anything, mock.Anything, mock.Anything).Return(nil, exception) + err := es.eventhandler.OnLocalTimeout() + require.ErrorIs(es.T(), err, exception, "expect a wrapped exception") + }) + es.notifier.AssertNotCalled(es.T(), "OnOwnTimeout", mock.Anything) +} + +// Test100Timeout tests that receiving 100 TCs for increasing ranks advances rounds +func (es *EventHandlerSuite) Test100Timeout() { + for i := 0; i < 100; i++ { + tc := helper.MakeTC(helper.WithTCRank(es.initRank + uint64(i))) + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + es.endRank++ + require.NoError(es.T(), err) + } + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestLeaderBuild100States tests scenario where leader builds 100 proposals one after another +func (es *EventHandlerSuite) TestLeaderBuild100States() { + require.Equal(es.T(), 1, len(es.forks.proposals), "expect Forks to contain only root state") + + // I'm the leader for the first rank + es.committee.leaders[es.initRank] = struct{}{} + + totalRank := 100 + for i := 0; i < totalRank; i++ { + // I'm the leader for 100 ranks + // I'm the next leader + es.committee.leaders[es.initRank+uint64(i+1)] = struct{}{} + // I can build qc for all 100 ranks + proposal := createProposal(es.initRank+uint64(i), es.initRank+uint64(i)-1) + qc := createQC(proposal.State) + + // for first proposal we need to store the parent otherwise it won't be voted for + if i == 0 { + parentState := helper.MakeState(func(state *models.State[*helper.TestState]) { + state.Identifier = proposal.State.ParentQuorumCertificate.GetSelector() + state.Rank = proposal.State.ParentQuorumCertificate.GetRank() + }) + es.forks.proposals[parentState.Identifier] = parentState + } + + es.safetyRules.votable[proposal.State.Identifier] = struct{}{} + // should trigger 100 rank change + es.endRank++ + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + ownProposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + require.Equal(es.T(), proposal.State.Rank+1, ownProposal.State.Rank) + }).Once() + vote := &helper.TestVote{ + Rank: proposal.State.Rank, + StateID: proposal.State.Identifier, + } + es.notifier.On("OnOwnVote", mock.MatchedBy(func(v **helper.TestVote) bool { return vote.Rank == (*v).Rank && vote.StateID == (*v).StateID }), mock.Anything).Once() + + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + err = es.eventhandler.OnReceiveQuorumCertificate(qc) + require.NoError(es.T(), err) + } + + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + require.Equal(es.T(), totalRank+1, len(es.forks.proposals), "expect Forks to contain root state + 100 proposed states") + es.notifier.AssertExpectations(es.T()) +} + +// TestFollowerFollows100States tests scenario where follower receives 100 proposals one after another +func (es *EventHandlerSuite) TestFollowerFollows100States() { + // add parent proposal otherwise we can't propose + parentProposal := createProposal(es.initRank, es.initRank-1) + es.forks.proposals[parentProposal.State.Identifier] = parentProposal.State + for i := 0; i < 100; i++ { + // create each proposal as if they are created by some leader + proposal := createProposal(es.initRank+uint64(i)+1, es.initRank+uint64(i)) + // as a follower, I receive these proposals + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + es.endRank++ + } + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + require.Equal(es.T(), 100, len(es.forks.proposals)-2) +} + +// TestFollowerReceives100Forks tests scenario where follower receives 100 forks built on top of the same state +func (es *EventHandlerSuite) TestFollowerReceives100Forks() { + for i := 0; i < 100; i++ { + // create each proposal as if they are created by some leader + proposal := createProposal(es.initRank+uint64(i)+1, es.initRank-1) + proposal.PreviousRankTimeoutCertificate = helper.MakeTC(helper.WithTCRank(es.initRank+uint64(i)), + helper.WithTCNewestQC(proposal.State.ParentQuorumCertificate)) + // expect a rank change since fork can be made only if last rank has ended with TC. + es.endRank++ + // as a follower, I receive these proposals + err := es.eventhandler.OnReceiveProposal(proposal) + require.NoError(es.T(), err) + } + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + require.Equal(es.T(), 100, len(es.forks.proposals)-1) +} + +// TestStart_ProposeOnce tests that after starting event handler we don't create proposal in case we have already proposed +// for this rank. +func (es *EventHandlerSuite) TestStart_ProposeOnce() { + // I'm the next leader + es.committee.leaders[es.initRank+1] = struct{}{} + es.endRank++ + + // STEP 1: simulating events _before_ a crash: EventHandler receives proposal and then a QC for the proposal (from VoteAggregator) + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Once() + err := es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + // constructing QC triggers making state proposal + err = es.eventhandler.OnReceiveQuorumCertificate(es.qc) + require.NoError(es.T(), err) + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) + + // Here, a hypothetical crash would happen. + // During crash recovery, Forks and Pacemaker are recovered to have exactly the same in-memory state as before + // Start triggers proposing logic. But as our own proposal for the rank is already in Forks, we should not propose again. + err = es.eventhandler.Start(es.ctx) + require.NoError(es.T(), err) + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + + // assert that broadcast wasn't trigger again, i.e. there should have been only one event `OnOwnProposal` in total + es.notifier.AssertNumberOfCalls(es.T(), "OnOwnProposal", 1) +} + +// TestCreateProposal_SanityChecks tests that proposing logic performs sanity checks when creating new state proposal. +// Specifically it tests a case where TC contains QC which: TC.Rank == TC.NewestQC.Rank +func (es *EventHandlerSuite) TestCreateProposal_SanityChecks() { + // round ended with TC where TC.Rank == TC.NewestQC.Rank + tc := helper.MakeTC(helper.WithTCRank(es.initRank), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCState(es.votingProposal.State)))) + + es.forks.proposals[es.votingProposal.State.Identifier] = es.votingProposal.State + + // I'm the next leader + es.committee.leaders[tc.GetRank()+1] = struct{}{} + + es.notifier.On("OnOwnProposal", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + proposal, ok := args[0].(*models.SignedProposal[*helper.TestState, *helper.TestVote]) + require.True(es.T(), ok) + // we need to make sure that produced proposal contains only QC even if there is TC for previous rank as well + require.Nil(es.T(), proposal.PreviousRankTimeoutCertificate) + }).Once() + + err := es.eventhandler.OnReceiveTimeoutCertificate(tc) + require.NoError(es.T(), err) + + require.Equal(es.T(), tc.GetLatestQuorumCert(), es.paceMaker.LatestQuorumCertificate()) + require.Equal(es.T(), tc, es.paceMaker.PriorRankTimeoutCertificate()) + require.Equal(es.T(), tc.GetRank()+1, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnReceiveProposal_ProposalForActiveRank tests that when receiving proposal for active we don't attempt to create a proposal +// Receiving proposal can trigger proposing logic only in case we have received missing state for past ranks. +func (es *EventHandlerSuite) TestOnReceiveProposal_ProposalForActiveRank() { + // receive proposal where we are leader, meaning that we have produced this proposal + es.committee.leaders[es.votingProposal.State.Rank] = struct{}{} + + err := es.eventhandler.OnReceiveProposal(es.votingProposal) + require.NoError(es.T(), err) + + es.notifier.AssertNotCalled(es.T(), "OnOwnProposal", mock.Anything, mock.Anything) +} + +// TestOnPartialTimeoutCertificateCreated_ProducedTimeout tests that when receiving partial TC for active rank we will create a timeout state +// immediately. +func (es *EventHandlerSuite) TestOnPartialTimeoutCertificateCreated_ProducedTimeout() { + partialTimeoutCertificate := &consensus.PartialTimeoutCertificateCreated{ + Rank: es.initRank, + NewestQuorumCertificate: es.parentProposal.State.ParentQuorumCertificate, + PriorRankTimeoutCertificate: nil, + } + + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutState, ok := args[0].(*models.TimeoutState[*helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a TO with same rank as partialTimeoutCertificate.Rank + require.Equal(es.T(), partialTimeoutCertificate.Rank, timeoutState.Rank) + }).Once() + + err := es.eventhandler.OnPartialTimeoutCertificateCreated(partialTimeoutCertificate) + require.NoError(es.T(), err) + + // partial TC shouldn't trigger rank change + require.Equal(es.T(), partialTimeoutCertificate.Rank, es.paceMaker.CurrentRank(), "incorrect rank change") +} + +// TestOnPartialTimeoutCertificateCreated_NotActiveRank tests that we don't create timeout state if partial TC was delivered for a past, non-current rank. +// NOTE: it is not possible to receive a partial timeout for a FUTURE rank, unless the partial timeout contains +// either a QC/TC allowing us to enter that rank, therefore that case is not covered here. +// See TestOnPartialTimeoutCertificateCreated_QcAndTimeoutCertificateProcessing instead. +func (es *EventHandlerSuite) TestOnPartialTimeoutCertificateCreated_NotActiveRank() { + partialTimeoutCertificate := &consensus.PartialTimeoutCertificateCreated{ + Rank: es.initRank - 1, + NewestQuorumCertificate: es.parentProposal.State.ParentQuorumCertificate, + } + + err := es.eventhandler.OnPartialTimeoutCertificateCreated(partialTimeoutCertificate) + require.NoError(es.T(), err) + + // partial TC shouldn't trigger rank change + require.Equal(es.T(), es.initRank, es.paceMaker.CurrentRank(), "incorrect rank change") + // we don't want to create timeout if partial TC was delivered for rank different than active one. + es.notifier.AssertNotCalled(es.T(), "OnOwnTimeout", mock.Anything) +} + +// TestOnPartialTimeoutCertificateCreated_QcAndTimeoutCertificateProcessing tests that EventHandler processes QC and TC included in consensus.PartialTimeoutCertificateCreated +// data structure. This tests cases like the following example: +// * the pacemaker is in rank 10 +// * we observe a partial timeout for rank 11 with a QC for rank 10 +// * we should change to rank 11 using the QC, then broadcast a timeout for rank 11 +func (es *EventHandlerSuite) TestOnPartialTimeoutCertificateCreated_QcAndTimeoutCertificateProcessing() { + + testOnPartialTimeoutCertificateCreated := func(partialTimeoutCertificate *consensus.PartialTimeoutCertificateCreated) { + es.endRank++ + + es.notifier.On("OnOwnTimeout", mock.Anything).Run(func(args mock.Arguments) { + timeoutState, ok := args[0].(*models.TimeoutState[*helper.TestVote]) + require.True(es.T(), ok) + // it should broadcast a TO with same rank as partialTimeoutCertificate.Rank + require.Equal(es.T(), partialTimeoutCertificate.Rank, timeoutState.Rank) + }).Once() + + err := es.eventhandler.OnPartialTimeoutCertificateCreated(partialTimeoutCertificate) + require.NoError(es.T(), err) + + require.Equal(es.T(), es.endRank, es.paceMaker.CurrentRank(), "incorrect rank change") + } + + es.Run("qc-triggered-rank-change", func() { + partialTimeoutCertificate := &consensus.PartialTimeoutCertificateCreated{ + Rank: es.qc.GetRank() + 1, + NewestQuorumCertificate: es.qc, + } + testOnPartialTimeoutCertificateCreated(partialTimeoutCertificate) + }) + es.Run("tc-triggered-rank-change", func() { + tc := helper.MakeTC(helper.WithTCRank(es.endRank), helper.WithTCNewestQC(es.qc)) + partialTimeoutCertificate := &consensus.PartialTimeoutCertificateCreated{ + Rank: tc.GetRank() + 1, + NewestQuorumCertificate: tc.GetLatestQuorumCert(), + PriorRankTimeoutCertificate: tc, + } + testOnPartialTimeoutCertificateCreated(partialTimeoutCertificate) + }) +} + +func createState(rank uint64) *models.State[*helper.TestState] { + return &models.State[*helper.TestState]{ + Identifier: fmt.Sprintf("%d", rank), + Rank: rank, + } +} + +func createStateWithQC(rank uint64, qcrank uint64) *models.State[*helper.TestState] { + state := createState(rank) + parent := createState(qcrank) + state.ParentQuorumCertificate = createQC(parent) + return state +} + +func createQC(parent *models.State[*helper.TestState]) models.QuorumCertificate { + qc := &helper.TestQuorumCertificate{ + Selector: parent.Identifier, + Rank: parent.Rank, + FrameNumber: parent.Rank, + AggregatedSignature: &helper.TestAggregatedSignature{ + Signature: make([]byte, 74), + Bitmask: []byte{0x1}, + PublicKey: make([]byte, 585), + }, + } + return qc +} + +func createVote(state *models.State[*helper.TestState]) *helper.TestVote { + return &helper.TestVote{ + Rank: state.Rank, + StateID: state.Identifier, + ID: "0", + Signature: make([]byte, 74), + } +} + +func createProposal(rank uint64, qcrank uint64) *models.SignedProposal[*helper.TestState, *helper.TestVote] { + state := createStateWithQC(rank, qcrank) + return helper.MakeSignedProposal[*helper.TestState, *helper.TestVote]( + helper.WithProposal[*helper.TestState, *helper.TestVote]( + helper.MakeProposal(helper.WithState(state)))) +} diff --git a/consensus/event_loop/event_loop.go b/consensus/eventloop/event_loop.go similarity index 78% rename from consensus/event_loop/event_loop.go rename to consensus/eventloop/event_loop.go index 8beafb9..438114a 100644 --- a/consensus/event_loop/event_loop.go +++ b/consensus/eventloop/event_loop.go @@ -22,17 +22,17 @@ type queuedProposal[StateT models.Unique, VoteT models.Unique] struct { // EventLoop buffers all incoming events to the hotstuff EventHandler, and feeds // EventHandler one event at a time. type EventLoop[StateT models.Unique, VoteT models.Unique] struct { - ctx context.Context - eventHandler consensus.EventHandler[StateT, VoteT] - proposals chan queuedProposal[StateT, VoteT] - newestSubmittedTc *tracker.NewestTCTracker - newestSubmittedQc *tracker.NewestQCTracker - newestSubmittedPartialTc *tracker.NewestPartialTcTracker - tcSubmittedNotifier chan struct{} - qcSubmittedNotifier chan struct{} - partialTcCreatedNotifier chan struct{} - startTime time.Time - tracer consensus.TraceLogger + ctx context.Context + eventHandler consensus.EventHandler[StateT, VoteT] + proposals chan queuedProposal[StateT, VoteT] + newestSubmittedTimeoutCertificate *tracker.NewestTCTracker + newestSubmittedQc *tracker.NewestQCTracker + newestSubmittedPartialTimeoutCertificate *tracker.NewestPartialTimeoutCertificateTracker + tcSubmittedNotifier chan struct{} + qcSubmittedNotifier chan struct{} + partialTimeoutCertificateCreatedNotifier chan struct{} + startTime time.Time + tracer consensus.TraceLogger } var _ consensus.EventLoop[*nilUnique, *nilUnique] = (*EventLoop[*nilUnique, *nilUnique])(nil) @@ -53,16 +53,16 @@ func NewEventLoop[StateT models.Unique, VoteT models.Unique]( proposals := make(chan queuedProposal[StateT, VoteT], 1000) el := &EventLoop[StateT, VoteT]{ - tracer: tracer, - eventHandler: eventHandler, - proposals: proposals, - tcSubmittedNotifier: make(chan struct{}, 1), - qcSubmittedNotifier: make(chan struct{}, 1), - partialTcCreatedNotifier: make(chan struct{}, 1), - newestSubmittedTc: tracker.NewNewestTCTracker(), - newestSubmittedQc: tracker.NewNewestQCTracker(), - newestSubmittedPartialTc: tracker.NewNewestPartialTcTracker(), - startTime: startTime, + tracer: tracer, + eventHandler: eventHandler, + proposals: proposals, + tcSubmittedNotifier: make(chan struct{}, 1), + qcSubmittedNotifier: make(chan struct{}, 1), + partialTimeoutCertificateCreatedNotifier: make(chan struct{}, 1), + newestSubmittedTimeoutCertificate: tracker.NewNewestTCTracker(), + newestSubmittedQc: tracker.NewNewestQCTracker(), + newestSubmittedPartialTimeoutCertificate: tracker.NewNewestPartialTimeoutCertificateTracker(), + startTime: startTime, } return el, nil @@ -71,17 +71,19 @@ func NewEventLoop[StateT models.Unique, VoteT models.Unique]( func (el *EventLoop[StateT, VoteT]) Start(ctx context.Context) error { el.ctx = ctx - select { - case <-ctx.Done(): - return nil - case <-time.After(time.Until(el.startTime)): - el.tracer.Trace("starting event loop") - err := el.loop(ctx) - if err != nil { - el.tracer.Error("irrecoverable event loop error", err) - return err + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(time.Until(el.startTime)): + el.tracer.Trace("starting event loop") + err := el.loop(ctx) + if err != nil { + el.tracer.Error("irrecoverable event loop error", err) + return + } } - } + }() return nil } @@ -102,7 +104,7 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { shutdownSignaled := ctx.Done() timeoutCertificates := el.tcSubmittedNotifier quorumCertificates := el.qcSubmittedNotifier - partialTCs := el.partialTcCreatedNotifier + partialTCs := el.partialTimeoutCertificateCreatedNotifier for { // Giving timeout events the priority to be processed first. @@ -116,12 +118,14 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { // if we receive the shutdown signal, exit the loop case <-shutdownSignaled: + el.tracer.Trace("shutting down event loop") return nil // processing timeout or partial TC event are top priority since // they allow node to contribute to TC aggregation when replicas can't // make progress on happy path case <-timeoutChannel: + el.tracer.Trace("received timeout") err = el.eventHandler.OnLocalTimeout() if err != nil { return fmt.Errorf("could not process timeout: %w", err) @@ -136,8 +140,9 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { continue case <-partialTCs: + el.tracer.Trace("received partial timeout") err = el.eventHandler.OnPartialTimeoutCertificateCreated( - el.newestSubmittedPartialTc.NewestPartialTc(), + el.newestSubmittedPartialTimeoutCertificate.NewestPartialTimeoutCertificate(), ) if err != nil { return fmt.Errorf("could not process partial created TC event: %w", err) @@ -153,6 +158,8 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { continue default: + el.tracer.Trace("non-priority event") + // fall through to non-priority events } @@ -161,10 +168,13 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { // same as before case <-shutdownSignaled: + el.tracer.Trace("shutting down event loop") return nil // same as before case <-timeoutChannel: + el.tracer.Trace("received timeout") + err = el.eventHandler.OnLocalTimeout() if err != nil { return fmt.Errorf("could not process timeout: %w", err) @@ -172,6 +182,8 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { // if we have a new proposal, process it case queuedItem := <-el.proposals: + el.tracer.Trace("received proposal") + proposal := queuedItem.proposal err = el.eventHandler.OnReceiveProposal(proposal) if err != nil { @@ -186,6 +198,7 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { // if we have a new QC, process it case <-quorumCertificates: + el.tracer.Trace("received quorum certificate") err = el.eventHandler.OnReceiveQuorumCertificate( *el.newestSubmittedQc.NewestQC(), ) @@ -195,16 +208,18 @@ func (el *EventLoop[StateT, VoteT]) loop(ctx context.Context) error { // if we have a new TC, process it case <-timeoutCertificates: + el.tracer.Trace("received timeout certificate") err = el.eventHandler.OnReceiveTimeoutCertificate( - *el.newestSubmittedTc.NewestTC(), + *el.newestSubmittedTimeoutCertificate.NewestTC(), ) if err != nil { return fmt.Errorf("could not process TC: %w", err) } case <-partialTCs: + el.tracer.Trace("received partial timeout certificate") err = el.eventHandler.OnPartialTimeoutCertificateCreated( - el.newestSubmittedPartialTc.NewestPartialTc(), + el.newestSubmittedPartialTimeoutCertificate.NewestPartialTimeoutCertificate(), ) if err != nil { return fmt.Errorf("could no process partial created TC event: %w", err) @@ -239,7 +254,7 @@ func (el *EventLoop[StateT, VoteT]) onTrustedQC(qc *models.QuorumCertificate) { // onTrustedTC pushes the received TC (which MUST be validated) to the // timeoutCertificates channel func (el *EventLoop[StateT, VoteT]) onTrustedTC(tc *models.TimeoutCertificate) { - if el.newestSubmittedTc.Track(tc) { + if el.newestSubmittedTimeoutCertificate.Track(tc) { el.tcSubmittedNotifier <- struct{}{} } else { qc := (*tc).GetLatestQuorumCert() @@ -257,8 +272,8 @@ func (el *EventLoop[StateT, VoteT]) OnTimeoutCertificateConstructedFromTimeouts( el.onTrustedTC(&tc) } -// OnPartialTimeoutCertificateCreated created a consensus.PartialTcCreated -// payload and pushes it into partialTcCreated buffered channel for further +// OnPartialTimeoutCertificateCreated created a consensus.PartialTimeoutCertificateCreated +// payload and pushes it into partialTimeoutCertificateCreated buffered channel for further // processing by EventHandler. Since we use buffered channel this function can // block if buffer is full. func (el *EventLoop[StateT, VoteT]) OnPartialTimeoutCertificateCreated( @@ -271,8 +286,8 @@ func (el *EventLoop[StateT, VoteT]) OnPartialTimeoutCertificateCreated( NewestQuorumCertificate: newestQC, PriorRankTimeoutCertificate: previousRankTimeoutCert, } - if el.newestSubmittedPartialTc.Track(event) { - el.partialTcCreatedNotifier <- struct{}{} + if el.newestSubmittedPartialTimeoutCertificate.Track(event) { + el.partialTimeoutCertificateCreatedNotifier <- struct{}{} } } diff --git a/consensus/eventloop/event_loop_test.go b/consensus/eventloop/event_loop_test.go new file mode 100644 index 0000000..c7be440 --- /dev/null +++ b/consensus/eventloop/event_loop_test.go @@ -0,0 +1,250 @@ +package eventloop + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +// TestEventLoop performs unit testing of event loop, checks if submitted events are propagated +// to event handler as well as handling of timeouts. +func TestEventLoop(t *testing.T) { + suite.Run(t, new(EventLoopTestSuite)) +} + +type EventLoopTestSuite struct { + suite.Suite + + eh *mocks.EventHandler[*helper.TestState, *helper.TestVote] + cancel context.CancelFunc + + eventLoop *EventLoop[*helper.TestState, *helper.TestVote] +} + +func (s *EventLoopTestSuite) SetupTest() { + s.eh = mocks.NewEventHandler[*helper.TestState, *helper.TestVote](s.T()) + s.eh.On("Start", mock.Anything).Return(nil).Maybe() + s.eh.On("TimeoutChannel").Return(make(<-chan time.Time, 1)).Maybe() + s.eh.On("OnLocalTimeout").Return(nil).Maybe() + + eventLoop, err := NewEventLoop(helper.Logger(), s.eh, time.Time{}) + require.NoError(s.T(), err) + s.eventLoop = eventLoop + + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + signalerCtx := ctx + + s.eventLoop.Start(signalerCtx) +} + +func (s *EventLoopTestSuite) TearDownTest() { + s.cancel() +} + +// TestReadyDone tests if event loop stops internal worker thread +func (s *EventLoopTestSuite) TestReadyDone() { + time.Sleep(1 * time.Second) + go func() { + s.cancel() + }() +} + +// Test_SubmitQC tests that submitted proposal is eventually sent to event handler for processing +func (s *EventLoopTestSuite) Test_SubmitProposal() { + proposal := helper.MakeSignedProposal[*helper.TestState, *helper.TestVote]() + processed := atomic.NewBool(false) + s.eh.On("OnReceiveProposal", proposal).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + s.eventLoop.SubmitProposal(proposal) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) +} + +// Test_SubmitQC tests that submitted QC is eventually sent to `EventHandler.OnReceiveQuorumCertificate` for processing +func (s *EventLoopTestSuite) Test_SubmitQC() { + // qcIngestionFunction is the archetype for EventLoop.OnQuorumCertificateConstructedFromVotes and EventLoop.OnNewQuorumCertificateDiscovered + type qcIngestionFunction func(models.QuorumCertificate) + + testQCIngestionFunction := func(f qcIngestionFunction, qcRank uint64) { + qc := helper.MakeQC(helper.WithQCRank(qcRank)) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveQuorumCertificate", qc).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(qc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + s.Run("QCs handed to EventLoop.OnQuorumCertificateConstructedFromVotes are forwarded to EventHandler", func() { + testQCIngestionFunction(s.eventLoop.OnQuorumCertificateConstructedFromVotes, 100) + }) + + s.Run("QCs handed to EventLoop.OnNewQuorumCertificateDiscovered are forwarded to EventHandler", func() { + testQCIngestionFunction(s.eventLoop.OnNewQuorumCertificateDiscovered, 101) + }) +} + +// Test_SubmitTC tests that submitted TC is eventually sent to `EventHandler.OnReceiveTimeoutCertificate` for processing +func (s *EventLoopTestSuite) Test_SubmitTC() { + // tcIngestionFunction is the archetype for EventLoop.OnTimeoutCertificateConstructedFromTimeouts and EventLoop.OnNewTimeoutCertificateDiscovered + type tcIngestionFunction func(models.TimeoutCertificate) + + testTCIngestionFunction := func(f tcIngestionFunction, tcRank uint64) { + tc := helper.MakeTC(helper.WithTCRank(tcRank)) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveTimeoutCertificate", tc).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(tc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + s.Run("TCs handed to EventLoop.OnTimeoutCertificateConstructedFromTimeouts are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnTimeoutCertificateConstructedFromTimeouts, 100) + }) + + s.Run("TCs handed to EventLoop.OnNewTimeoutCertificateDiscovered are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnNewTimeoutCertificateDiscovered, 101) + }) +} + +// Test_SubmitTC_IngestNewestQC tests that included QC in TC is eventually sent to `EventHandler.OnReceiveQuorumCertificate` for processing +func (s *EventLoopTestSuite) Test_SubmitTC_IngestNewestQC() { + // tcIngestionFunction is the archetype for EventLoop.OnTimeoutCertificateConstructedFromTimeouts and EventLoop.OnNewTimeoutCertificateDiscovered + type tcIngestionFunction func(models.TimeoutCertificate) + + testTCIngestionFunction := func(f tcIngestionFunction, tcRank, qcRank uint64) { + tc := helper.MakeTC(helper.WithTCRank(tcRank), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCRank(qcRank)))) + processed := atomic.NewBool(false) + s.eh.On("OnReceiveQuorumCertificate", tc.GetLatestQuorumCert()).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + f(tc) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) + } + + // process initial TC, this will track the newest TC + s.eh.On("OnReceiveTimeoutCertificate", mock.Anything).Return(nil).Once() + s.eventLoop.OnTimeoutCertificateConstructedFromTimeouts(helper.MakeTC( + helper.WithTCRank(100), + helper.WithTCNewestQC( + helper.MakeQC( + helper.WithQCRank(80), + ), + ), + )) + + s.Run("QCs handed to EventLoop.OnTimeoutCertificateConstructedFromTimeouts are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnTimeoutCertificateConstructedFromTimeouts, 100, 99) + }) + + s.Run("QCs handed to EventLoop.OnNewTimeoutCertificateDiscovered are forwarded to EventHandler", func() { + testTCIngestionFunction(s.eventLoop.OnNewTimeoutCertificateDiscovered, 100, 100) + }) +} + +// Test_OnPartialTimeoutCertificateCreated tests that event loop delivers partialTimeoutCertificateCreated events to event handler. +func (s *EventLoopTestSuite) Test_OnPartialTimeoutCertificateCreated() { + rank := uint64(1000) + newestQC := helper.MakeQC(helper.WithQCRank(rank - 10)) + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(rank-1), helper.WithTCNewestQC(newestQC)) + + processed := atomic.NewBool(false) + partialTimeoutCertificateCreated := &consensus.PartialTimeoutCertificateCreated{ + Rank: rank, + NewestQuorumCertificate: newestQC, + PriorRankTimeoutCertificate: previousRankTimeoutCert, + } + s.eh.On("OnPartialTimeoutCertificateCreated", partialTimeoutCertificateCreated).Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + s.eventLoop.OnPartialTimeoutCertificateCreated(rank, newestQC, previousRankTimeoutCert) + require.Eventually(s.T(), processed.Load, time.Millisecond*100, time.Millisecond*10) +} + +// TestEventLoop_Timeout tests that event loop delivers timeout events to event handler under pressure +func TestEventLoop_Timeout(t *testing.T) { + eh := &mocks.EventHandler[*helper.TestState, *helper.TestVote]{} + processed := atomic.NewBool(false) + eh.On("Start", mock.Anything).Return(nil).Once() + eh.On("OnReceiveQuorumCertificate", mock.Anything).Return(nil).Maybe() + eh.On("OnReceiveProposal", mock.Anything).Return(nil).Maybe() + eh.On("OnLocalTimeout").Run(func(args mock.Arguments) { + processed.Store(true) + }).Return(nil).Once() + + eventLoop, err := NewEventLoop(helper.Logger(), eh, time.Time{}) + require.NoError(t, err) + + eh.On("TimeoutChannel").Return(time.After(100 * time.Millisecond)) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := ctx + eventLoop.Start(signalerCtx) + + time.Sleep(10 * time.Millisecond) + + var wg sync.WaitGroup + wg.Add(2) + + // spam with proposals and QCs + go func() { + defer wg.Done() + for !processed.Load() { + qc := helper.MakeQC() + eventLoop.OnQuorumCertificateConstructedFromVotes(qc) + } + }() + + go func() { + defer wg.Done() + for !processed.Load() { + eventLoop.SubmitProposal(helper.MakeSignedProposal[*helper.TestState, *helper.TestVote]()) + } + }() + + require.Eventually(t, processed.Load, time.Millisecond*200, time.Millisecond*10) + + cancel() +} + +// TestReadyDoneWithStartTime tests that event loop correctly starts and schedules start of processing +// when startTime argument is used +func TestReadyDoneWithStartTime(t *testing.T) { + eh := &mocks.EventHandler[*helper.TestState, *helper.TestVote]{} + eh.On("Start", mock.Anything).Return(nil) + eh.On("TimeoutChannel").Return(make(<-chan time.Time, 1)) + eh.On("OnLocalTimeout").Return(nil) + + startTimeDuration := 2 * time.Second + startTime := time.Now().Add(startTimeDuration) + eventLoop, err := NewEventLoop(helper.Logger(), eh, startTime) + require.NoError(t, err) + + done := make(chan struct{}) + eh.On("OnReceiveProposal", mock.AnythingOfType("*models.SignedProposal")).Run(func(args mock.Arguments) { + require.True(t, time.Now().After(startTime)) + close(done) + }).Return(nil).Once() + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := ctx + eventLoop.Start(signalerCtx) + + eventLoop.SubmitProposal(helper.MakeSignedProposal[*helper.TestState, *helper.TestVote]()) + + cancel() +} diff --git a/consensus/example/generic_consensus_example.go b/consensus/example/generic_consensus_example.go index db4ce96..94a5cc2 100644 --- a/consensus/example/generic_consensus_example.go +++ b/consensus/example/generic_consensus_example.go @@ -4,18 +4,20 @@ import ( "context" "errors" "fmt" + "log" "slices" "sync" "time" - "go.uber.org/zap" "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" + "source.quilibrium.com/quilibrium/monorepo/consensus/pacemaker" ) // Example using the generic state machine from the consensus package -// ConsensusData represents the state data -type ConsensusData struct { +// consensusData represents the state data +type consensusData struct { Round uint64 Hash string Votes map[string]interface{} @@ -26,16 +28,16 @@ type ConsensusData struct { } // Identity implements Unique interface -func (c ConsensusData) Identity() consensus.Identity { +func (c consensusData) Identity() models.Identity { return fmt.Sprintf("%s-%d", c.Hash, c.Round) } -func (c ConsensusData) Rank() uint64 { +func (c consensusData) GetRank() uint64 { return c.Round } -func (c ConsensusData) Clone() consensus.Unique { - return ConsensusData{ +func (c consensusData) Clone() models.Unique { + return consensusData{ Round: c.Round, Hash: c.Hash, Votes: c.Votes, @@ -47,7 +49,7 @@ func (c ConsensusData) Clone() consensus.Unique { } // Vote represents a vote in the consensus -type Vote struct { +type vote struct { NodeID string Round uint64 VoteValue string @@ -56,16 +58,16 @@ type Vote struct { } // Identity implements Unique interface -func (v Vote) Identity() consensus.Identity { +func (v vote) Identity() consensus.Identity { return fmt.Sprintf("%s-%d-%s", v.ProposerID, v.Round, v.VoteValue) } -func (v Vote) Rank() uint64 { +func (v vote) GetRank() uint64 { return v.Round } -func (v Vote) Clone() consensus.Unique { - return Vote{ +func (v vote) Clone() models.Unique { + return vote{ NodeID: v.NodeID, Round: v.Round, VoteValue: v.VoteValue, @@ -74,21 +76,113 @@ func (v Vote) Clone() consensus.Unique { } } -// PeerID represents a peer identifier -type PeerID struct { +type aggregateSignature struct { + Signature []byte + PublicKey []byte + Bitmask []byte +} + +// GetBitmask implements models.AggregatedSignature. +func (b *aggregateSignature) GetBitmask() []byte { + return b.Bitmask +} + +// GetPublicKey implements models.AggregatedSignature. +func (b *aggregateSignature) GetPublicKey() []byte { + return b.PublicKey +} + +// GetSignature implements models.AggregatedSignature. +func (b *aggregateSignature) GetSignature() []byte { + return b.Signature +} + +type quorumCertificate struct { + Filter []byte + Rank uint64 + FrameNumber uint64 + Selector []byte + Timestamp int64 + AggregatedSignature *aggregateSignature +} + +// GetAggregatedSignature implements models.QuorumCertificate. +func (q *quorumCertificate) GetAggregatedSignature() models.AggregatedSignature { + return q.AggregatedSignature +} + +// GetFilter implements models.QuorumCertificate. +func (q *quorumCertificate) GetFilter() []byte { + return q.Filter +} + +// GetFrameNumber implements models.QuorumCertificate. +func (q *quorumCertificate) GetFrameNumber() uint64 { + return q.FrameNumber +} + +// GetRank implements models.QuorumCertificate. +func (q *quorumCertificate) GetRank() uint64 { + return q.Rank +} + +// GetSelector implements models.QuorumCertificate. +func (q *quorumCertificate) GetSelector() []byte { + return q.Selector +} + +// GetTimestamp implements models.QuorumCertificate. +func (q *quorumCertificate) GetTimestamp() int64 { + return q.Timestamp +} + +var _ models.AggregatedSignature = (*aggregateSignature)(nil) +var _ models.QuorumCertificate = (*quorumCertificate)(nil) + +type store struct { + consensusState *models.ConsensusState + livenessState *models.LivenessState +} + +// GetConsensusState implements consensus.ConsensusStore. +func (s *store) GetConsensusState() (*models.ConsensusState, error) { + return s.consensusState, nil +} + +// GetLivenessState implements consensus.ConsensusStore. +func (s *store) GetLivenessState() (*models.LivenessState, error) { + return s.livenessState, nil +} + +// PutConsensusState implements consensus.ConsensusStore. +func (s *store) PutConsensusState(state *models.ConsensusState) error { + s.consensusState = state + return nil +} + +// PutLivenessState implements consensus.ConsensusStore. +func (s *store) PutLivenessState(state *models.LivenessState) error { + s.livenessState = state + return nil +} + +var _ consensus.ConsensusStore = (*store)(nil) + +// peerID represents a peer identifier +type peerID struct { ID string } // Identity implements Unique interface -func (p PeerID) Identity() consensus.Identity { +func (p peerID) Identity() consensus.Identity { return p.ID } -func (p PeerID) Rank() uint64 { +func (p peerID) GetRank() uint64 { return 0 } -func (p PeerID) Clone() consensus.Unique { +func (p peerID) Clone() models.Unique { return p } @@ -104,11 +198,11 @@ func (c CollectedData) Identity() consensus.Identity { return fmt.Sprintf("collected-%d", c.Timestamp.Unix()) } -func (c CollectedData) Rank() uint64 { +func (c CollectedData) GetRank() uint64 { return c.Round } -func (c CollectedData) Clone() consensus.Unique { +func (c CollectedData) Clone() models.Unique { return CollectedData{ Mutations: slices.Clone(c.Mutations), Timestamp: c.Timestamp, @@ -117,28 +211,27 @@ func (c CollectedData) Clone() consensus.Unique { // MockSyncProvider implements SyncProvider type MockSyncProvider struct { - logger *zap.Logger } func (m *MockSyncProvider) Synchronize( - existing *ConsensusData, ctx context.Context, -) (<-chan *ConsensusData, <-chan error) { - dataCh := make(chan *ConsensusData, 1) + existing *consensusData, +) (<-chan *consensusData, <-chan error) { + dataCh := make(chan *consensusData, 1) errCh := make(chan error, 1) go func() { defer close(dataCh) defer close(errCh) - m.logger.Info("synchronizing...") + log.Println("synchronizing...") select { case <-time.After(10 * time.Millisecond): - m.logger.Info("sync complete") + log.Println("sync complete") if existing != nil { dataCh <- existing } else { - dataCh <- &ConsensusData{ + dataCh <- &consensusData{ Round: 0, Hash: "genesis", Votes: make(map[string]interface{}), @@ -156,8 +249,7 @@ func (m *MockSyncProvider) Synchronize( // MockVotingProvider implements VotingProvider type MockVotingProvider struct { - logger *zap.Logger - votes map[string]*Vote + votes map[string]*vote currentRound uint64 voteTarget int mu sync.Mutex @@ -167,26 +259,22 @@ type MockVotingProvider struct { } func NewMockVotingProvider( - logger *zap.Logger, voteTarget int, nodeID string, ) *MockVotingProvider { return &MockVotingProvider{ - logger: logger, - votes: make(map[string]*Vote), + votes: make(map[string]*vote), voteTarget: voteTarget, nodeID: nodeID, } } func NewMaliciousVotingProvider( - logger *zap.Logger, voteTarget int, nodeID string, ) *MockVotingProvider { return &MockVotingProvider{ - logger: logger, - votes: make(map[string]*Vote), + votes: make(map[string]*vote), voteTarget: voteTarget, isMalicious: true, nodeID: nodeID, @@ -194,16 +282,18 @@ func NewMaliciousVotingProvider( } func (m *MockVotingProvider) SendProposal( - proposal *ConsensusData, ctx context.Context, + proposal *consensusData, ) error { - m.logger.Info("sending proposal", - zap.Uint64("round", proposal.Round), - zap.String("hash", proposal.Hash)) + log.Printf( + "sending proposal, round: %d, hash: %s\n", + proposal.Round, + proposal.Hash, + ) if m.messageBus != nil { // Make a copy to avoid sharing pointers between nodes - proposalCopy := &ConsensusData{ + proposalCopy := &consensusData{ Round: proposal.Round, Hash: proposal.Hash, Votes: make(map[string]interface{}), @@ -228,16 +318,18 @@ func (m *MockVotingProvider) SendProposal( } func (m *MockVotingProvider) DecideAndSendVote( - proposals map[consensus.Identity]*ConsensusData, ctx context.Context, -) (PeerID, *Vote, error) { + proposals map[consensus.Identity]*consensusData, +) (peerID, *vote, error) { m.mu.Lock() defer m.mu.Unlock() // Log available proposals - m.logger.Info("deciding vote", - zap.Int("proposal_count", len(proposals)), - zap.String("node_id", m.nodeID)) + log.Printf( + "deciding vote, proposal count: %d, node id: %s\n", + len(proposals), + m.nodeID, + ) nodes := []string{ "prover-node-1", @@ -246,7 +338,7 @@ func (m *MockVotingProvider) DecideAndSendVote( "validator-node-3", } - var chosenProposal *ConsensusData + var chosenProposal *consensusData var chosenID consensus.Identity if len(proposals) > 3 { leaderIdx := int(proposals[nodes[0]].Round % uint64(len(nodes))) @@ -257,15 +349,17 @@ func (m *MockVotingProvider) DecideAndSendVote( chosenProposal = proposals[nodes[(leaderIdx+1)%len(nodes)]] chosenID = nodes[(leaderIdx+1)%len(nodes)] } - m.logger.Info("found proposal", - zap.String("from", chosenID), - zap.Uint64("round", chosenProposal.Round)) + log.Printf( + "found proposal, from: %s, round: %d\n", + chosenID, + chosenProposal.Round, + ) } if chosenProposal == nil { - return PeerID{}, nil, fmt.Errorf("no proposals to vote on") + return peerID{}, nil, fmt.Errorf("no proposals to vote on") } - vote := &Vote{ + vt := &vote{ NodeID: m.nodeID, Round: chosenProposal.Round, VoteValue: "approve", @@ -273,21 +367,23 @@ func (m *MockVotingProvider) DecideAndSendVote( ProposerID: chosenID, } - m.votes[vote.NodeID] = vote - m.logger.Info("decided and sent vote", - zap.String("node_id", vote.NodeID), - zap.String("vote", vote.VoteValue), - zap.Uint64("round", vote.Round), - zap.String("for_proposal", chosenID)) + m.votes[vt.NodeID] = vt + log.Printf( + "decided and sent vote, node id: %s, vote: %s, round: %d, for proposal: %s\n", + vt.NodeID, + vt.VoteValue, + vt.Round, + chosenID, + ) if m.messageBus != nil { // Make a copy to avoid sharing pointers - voteCopy := &Vote{ - NodeID: vote.NodeID, - Round: vote.Round, - VoteValue: vote.VoteValue, - Timestamp: vote.Timestamp, - ProposerID: vote.ProposerID, + voteCopy := &vote{ + NodeID: vt.NodeID, + Round: vt.Round, + VoteValue: vt.VoteValue, + Timestamp: vt.Timestamp, + ProposerID: vt.ProposerID, } m.messageBus.Broadcast(Message{ Type: "vote", @@ -296,27 +392,29 @@ func (m *MockVotingProvider) DecideAndSendVote( }) } - return PeerID{ID: chosenID}, vote, nil + return peerID{ID: chosenID}, vt, nil } -func (m *MockVotingProvider) SendVote(vote *Vote, ctx context.Context) ( - PeerID, +func (m *MockVotingProvider) SendVote(ctx context.Context, vt *vote) ( + peerID, error, ) { - m.logger.Info("re-sent vote", - zap.String("node_id", vote.NodeID), - zap.String("vote", vote.VoteValue), - zap.Uint64("round", vote.Round), - zap.String("for_proposal", vote.ProposerID)) + log.Printf( + "re-sent vote, node id: %s, vote: %s, round: %d, for proposal: %s\n", + vt.NodeID, + vt.VoteValue, + vt.Round, + vt.ProposerID, + ) if m.messageBus != nil { // Make a copy to avoid sharing pointers - voteCopy := &Vote{ - NodeID: vote.NodeID, - Round: vote.Round, - VoteValue: vote.VoteValue, - Timestamp: vote.Timestamp, - ProposerID: vote.ProposerID, + voteCopy := &vote{ + NodeID: vt.NodeID, + Round: vt.Round, + VoteValue: vt.VoteValue, + Timestamp: vt.Timestamp, + ProposerID: vt.ProposerID, } m.messageBus.Broadcast(Message{ Type: "vote", @@ -324,18 +422,20 @@ func (m *MockVotingProvider) SendVote(vote *Vote, ctx context.Context) ( Data: voteCopy, }) } - return PeerID{ID: vote.ProposerID}, nil + return peerID{ID: vt.ProposerID}, nil } func (m *MockVotingProvider) IsQuorum( - proposalVotes map[consensus.Identity]*Vote, ctx context.Context, + proposalVotes map[consensus.Identity]*vote, ) (bool, error) { m.mu.Lock() defer m.mu.Unlock() - m.logger.Info("checking quorum", - zap.Int("target", m.voteTarget)) + log.Printf( + "checking quorum, target: %d\n", + m.voteTarget, + ) totalVotes := 0 fmt.Printf("%s %+v\n", m.nodeID, proposalVotes) voteCount := map[string]int{} @@ -360,16 +460,18 @@ func (m *MockVotingProvider) IsQuorum( } func (m *MockVotingProvider) FinalizeVotes( - proposals map[consensus.Identity]*ConsensusData, - proposalVotes map[consensus.Identity]*Vote, ctx context.Context, -) (*ConsensusData, PeerID, error) { + proposals map[consensus.Identity]*consensusData, + proposalVotes map[consensus.Identity]*vote, +) (*consensusData, peerID, error) { // Count approvals - m.logger.Info("finalizing votes", - zap.Int("total_proposals", len(proposals))) + log.Printf( + "finalizing votes, total proposals: %d\n", + len(proposals), + ) winnerCount := 0 - var winnerProposal *ConsensusData = nil - var winnerProposer PeerID + var winnerProposal *consensusData = nil + var winnerProposer peerID voteCount := map[string]int{} for _, votes := range proposalVotes { count, ok := voteCount[votes.ProposerID] @@ -379,7 +481,7 @@ func (m *MockVotingProvider) FinalizeVotes( voteCount[votes.ProposerID] = count + 1 } } - for peerID, proposal := range proposals { + for pid, proposal := range proposals { if proposal == nil { continue } @@ -387,16 +489,18 @@ func (m *MockVotingProvider) FinalizeVotes( if voteCount > winnerCount { winnerCount = voteCount winnerProposal = proposal - winnerProposer = PeerID{ID: peerID} + winnerProposer = peerID{ID: pid} } } - m.logger.Info("vote summary", - zap.Int("approvals", winnerCount), - zap.Int("required", m.voteTarget)) + log.Printf( + "vote summary, approvals: %d, required: %d\n", + winnerCount, + m.voteTarget, + ) if winnerCount < m.voteTarget { - return nil, PeerID{}, fmt.Errorf( + return nil, peerID{}, fmt.Errorf( "not enough approvals: %d < %d", winnerCount, m.voteTarget, @@ -410,7 +514,7 @@ func (m *MockVotingProvider) FinalizeVotes( // Pick the first proposal for id, prop := range proposals { // Create a new finalized state based on the chosen proposal - finalizedState := &ConsensusData{ + finalizedState := &consensusData{ Round: prop.Round, Hash: prop.Hash, Votes: make(map[string]interface{}), @@ -424,32 +528,36 @@ func (m *MockVotingProvider) FinalizeVotes( finalizedState.Votes[k] = v } - m.logger.Info("finalized state", - zap.Uint64("round", finalizedState.Round), - zap.String("hash", finalizedState.Hash), - zap.String("proposer", id)) - return finalizedState, PeerID{ID: id}, nil + log.Printf( + "finalized state, round: %d, hash: %s, proposer: %d\n", + finalizedState.Round, + finalizedState.Hash, + id, + ) + return finalizedState, peerID{ID: id}, nil } - return nil, PeerID{}, fmt.Errorf("no proposals to finalize") + return nil, peerID{}, fmt.Errorf("no proposals to finalize") } func (m *MockVotingProvider) SendConfirmation( - finalized *ConsensusData, ctx context.Context, + finalized *consensusData, ) error { if finalized == nil { - m.logger.Warn("cannot send confirmation for nil state") + log.Println("cannot send confirmation for nil state") return fmt.Errorf("cannot send confirmation for nil state") } - m.logger.Info("sending confirmation", - zap.Uint64("round", finalized.Round), - zap.String("hash", finalized.Hash)) + log.Printf( + "sending confirmation, round: %d, hash: %s\n", + finalized.Round, + finalized.Hash, + ) if m.messageBus != nil { // Make a copy to avoid sharing pointers - confirmationCopy := &ConsensusData{ + confirmationCopy := &consensusData{ Round: finalized.Round, Hash: finalized.Hash, Votes: make(map[string]interface{}), @@ -476,10 +584,10 @@ func (m *MockVotingProvider) SendConfirmation( func (m *MockVotingProvider) Reset() { m.mu.Lock() defer m.mu.Unlock() - m.votes = make(map[string]*Vote) - m.logger.Info( - "reset voting provider", - zap.Uint64("current_round", m.currentRound), + m.votes = make(map[string]*vote) + log.Printf( + "reset voting provider, current round: %d\n", + m.currentRound, ) } @@ -487,20 +595,19 @@ func (m *MockVotingProvider) SetRound(round uint64) { m.mu.Lock() defer m.mu.Unlock() m.currentRound = round - m.logger.Info("voting provider round updated", zap.Uint64("round", round)) + log.Printf("voting provider round updated, round: %d\n", round) } // MockLeaderProvider implements LeaderProvider type MockLeaderProvider struct { - logger *zap.Logger isProver bool nodeID string } func (m *MockLeaderProvider) GetNextLeaders( - prior *ConsensusData, ctx context.Context, -) ([]PeerID, error) { + prior *consensusData, +) ([]peerID, error) { // Simple round-robin leader selection round := uint64(0) if prior != nil { @@ -516,37 +623,32 @@ func (m *MockLeaderProvider) GetNextLeaders( // Select leader based on round leaderIdx := int(round % uint64(len(nodes))) - leaders := []PeerID{ + leaders := []peerID{ {ID: nodes[leaderIdx]}, {ID: nodes[uint64(leaderIdx+1)%uint64(len(nodes))]}, {ID: nodes[uint64(leaderIdx+2)%uint64(len(nodes))]}, {ID: nodes[uint64(leaderIdx+3)%uint64(len(nodes))]}, } - m.logger.Info("selected next leaders", - zap.Uint64("round", round), - zap.String("leader", leaders[0].ID)) + fmt.Printf( + "selected next leaders, round: %d, leader: %s\n", + round, + leaders[0].ID, + ) return leaders, nil } func (m *MockLeaderProvider) ProveNextState( - prior *ConsensusData, - collected CollectedData, ctx context.Context, -) (*ConsensusData, error) { + prior *consensusData, + collected CollectedData, +) (*consensusData, error) { priorRound := uint64(0) - priorHash := "genesis" if prior != nil { priorRound = prior.Round - priorHash = prior.Hash } - m.logger.Info("generating proof", - zap.Uint64("prior_round", priorRound), - zap.String("prior_hash", priorHash), - zap.Int("mutations", len(collected.Mutations))) - select { case <-time.After(500 * time.Millisecond): proof := map[string]interface{}{ @@ -555,7 +657,7 @@ func (m *MockLeaderProvider) ProveNextState( "prover": m.nodeID, } - newState := &ConsensusData{ + newState := &consensusData{ Round: priorRound + 1, Hash: fmt.Sprintf("block_%d", priorRound+1), Votes: make(map[string]interface{}), @@ -573,7 +675,6 @@ func (m *MockLeaderProvider) ProveNextState( // MockLivenessProvider implements LivenessProvider type MockLivenessProvider struct { - logger *zap.Logger round uint64 nodeID string messageBus *MessageBus @@ -582,7 +683,7 @@ type MockLivenessProvider struct { func (m *MockLivenessProvider) Collect( ctx context.Context, ) (CollectedData, error) { - m.logger.Info("collecting mutations") + fmt.Println("collecting mutations") // Simulate collecting some mutations mutations := []string{ @@ -599,18 +700,20 @@ func (m *MockLivenessProvider) Collect( } func (m *MockLivenessProvider) SendLiveness( - prior *ConsensusData, - collected CollectedData, ctx context.Context, + prior *consensusData, + collected CollectedData, ) error { round := uint64(0) if prior != nil { round = prior.Round } - m.logger.Info("sending liveness signal", - zap.Uint64("round", round), - zap.Int("mutations", len(collected.Mutations))) + fmt.Printf( + "sending liveness signal, round: %d, mutations: %d\n", + round, + len(collected.Mutations), + ) if m.messageBus != nil { // Make a copy to avoid sharing pointers @@ -633,13 +736,18 @@ func (m *MockLivenessProvider) SendLiveness( // ConsensusNode represents a node using the generic state machine type ConsensusNode struct { - sm *consensus.StateMachine[ - ConsensusData, - Vote, - PeerID, + p *pacemaker.Pacemaker[ + consensusData, + vote, + peerID, + CollectedData, + ] + sm *consensus.StateMachine[ + consensusData, + vote, + peerID, CollectedData, ] - logger *zap.Logger nodeID string ctx context.Context cancel context.CancelFunc @@ -655,13 +763,11 @@ func NewConsensusNode( nodeID string, isProver bool, voteTarget int, - logger *zap.Logger, ) *ConsensusNode { return newConsensusNodeWithBehavior( nodeID, isProver, voteTarget, - logger, false, ) } @@ -671,13 +777,11 @@ func NewMaliciousNode( nodeID string, isProver bool, voteTarget int, - logger *zap.Logger, ) *ConsensusNode { return newConsensusNodeWithBehavior( nodeID, isProver, voteTarget, - logger, true, ) } @@ -686,11 +790,10 @@ func newConsensusNodeWithBehavior( nodeID string, isProver bool, voteTarget int, - logger *zap.Logger, isMalicious bool, ) *ConsensusNode { // Create initial consensus data - initialData := &ConsensusData{ + initialData := &consensusData{ Round: 0, Hash: "genesis", Votes: make(map[string]interface{}), @@ -700,29 +803,27 @@ func newConsensusNodeWithBehavior( } // Create mock implementations - syncProvider := &MockSyncProvider{logger: logger} + syncProvider := &MockSyncProvider{} var votingProvider *MockVotingProvider if isMalicious { - votingProvider = NewMaliciousVotingProvider(logger, voteTarget, nodeID) + votingProvider = NewMaliciousVotingProvider(voteTarget, nodeID) } else { - votingProvider = NewMockVotingProvider(logger, voteTarget, nodeID) + votingProvider = NewMockVotingProvider(voteTarget, nodeID) } leaderProvider := &MockLeaderProvider{ - logger: logger, isProver: isProver, nodeID: nodeID, } livenessProvider := &MockLivenessProvider{ - logger: logger, nodeID: nodeID, } // Create the state machine - sm := consensus.NewStateMachine( - PeerID{ID: nodeID}, + sm := consensus.NewStateMachine[consensusData, vote, peerID, CollectedData]( + peerID{ID: nodeID}, initialData, true, func() uint64 { return uint64(3) }, @@ -730,14 +831,36 @@ func newConsensusNodeWithBehavior( votingProvider, leaderProvider, livenessProvider, - tracer{logger: logger}, + tracer{}, ) + p, err := pacemaker.NewPacemaker[consensusData, vote, peerID, CollectedData]( + peerID{ID: nodeID}, + func() *models.LivenessState { + return &models.LivenessState{ + Filter: nil, + CurrentRank: 0, + LatestQuorumCertificate: &quorumCertificate{}, + PriorRankTimeoutCertificate: nil, + } + }, + func() uint64 { return uint64(3) }, + votingProvider, + leaderProvider, + livenessProvider, + sm, + &store{}, + tracer{}, + ) + if err != nil { + panic(err) + } + ctx, cancel := context.WithCancel(context.Background()) node := &ConsensusNode{ + p: p, sm: sm, - logger: logger, nodeID: nodeID, ctx: ctx, cancel: cancel, @@ -748,42 +871,40 @@ func newConsensusNodeWithBehavior( // Add transition listener sm.AddListener(&NodeTransitionListener{ - logger: logger, - node: node, + node: node, }) return node } type tracer struct { - logger *zap.Logger } // Error implements consensus.TraceLogger. func (t tracer) Error(message string, err error) { - t.logger.Error(message, zap.Error(err)) + fmt.Println(message, err) } // Trace implements consensus.TraceLogger. func (t tracer) Trace(message string) { - t.logger.Debug(message) + fmt.Println(message) } // Start begins the consensus node func (n *ConsensusNode) Start() error { - n.logger.Info("starting consensus node", zap.String("node_id", n.nodeID)) + fmt.Printf("starting consensus node, node id: %s\n", n.nodeID) // Start monitoring for messages go n.monitor() - return n.sm.Start() + return n.p.Start(n.ctx) } // Stop halts the consensus node func (n *ConsensusNode) Stop() error { - n.logger.Info("stopping consensus node", zap.String("node_id", n.nodeID)) + fmt.Printf("stopping consensus node, node id: %s\n", n.nodeID) n.cancel() - return n.sm.Stop() + return nil } // SetMessageBus connects the node to the message bus @@ -814,38 +935,39 @@ func (n *ConsensusNode) monitor() { // handleMessage processes messages from other nodes func (n *ConsensusNode) handleMessage(msg Message) { - n.logger.Debug("received message", - zap.String("type", msg.Type), - zap.String("from", msg.Sender)) + fmt.Printf( + "received message, type: %s, from: %s\n", + msg.Type, + msg.Sender, + ) switch msg.Type { case "proposal": - if proposal, ok := msg.Data.(*ConsensusData); ok { - n.sm.ReceiveProposal(PeerID{ID: msg.Sender}, proposal) + if proposal, ok := msg.Data.(*consensusData); ok { + n.sm.ReceiveProposal(proposal.Round, peerID{ID: msg.Sender}, proposal) } case "vote": - if vote, ok := msg.Data.(*Vote); ok { + if vote, ok := msg.Data.(*vote); ok { n.sm.ReceiveVote( - PeerID{ID: vote.ProposerID}, - PeerID{ID: msg.Sender}, + peerID{ID: vote.ProposerID}, + peerID{ID: msg.Sender}, vote, ) } case "liveness_check": if collected, ok := msg.Data.(CollectedData); ok { - n.sm.ReceiveLivenessCheck(PeerID{ID: msg.Sender}, collected) + n.sm.ReceiveLivenessCheck(peerID{ID: msg.Sender}, collected) } case "confirmation": - if confirmation, ok := msg.Data.(*ConsensusData); ok { - n.sm.ReceiveConfirmation(PeerID{ID: msg.Sender}, confirmation) + if confirmation, ok := msg.Data.(*consensusData); ok { + n.sm.ReceiveConfirmation(peerID{ID: msg.Sender}, confirmation) } } } // NodeTransitionListener handles state transitions type NodeTransitionListener struct { - logger *zap.Logger - node *ConsensusNode + node *ConsensusNode } func (l *NodeTransitionListener) OnTransition( @@ -853,11 +975,13 @@ func (l *NodeTransitionListener) OnTransition( to consensus.State, event consensus.Event, ) { - l.logger.Info("state transition", - zap.String("node_id", l.node.nodeID), - zap.String("from", string(from)), - zap.String("to", string(to)), - zap.String("event", string(event))) + fmt.Printf( + "state transition, node id: %s, from: %s, to: %s, event: %s\n", + l.node.nodeID, + string(from), + string(to), + string(event), + ) // Handle state-specific actions switch to { @@ -878,9 +1002,9 @@ func (l *NodeTransitionListener) handleEnterVoting() { // Malicious nodes exhibit Byzantine behavior if l.node.isMalicious { - l.logger.Warn( - "MALICIOUS NODE: Executing Byzantine behavior", - zap.String("node_id", l.node.nodeID), + fmt.Printf( + "MALICIOUS NODE: Executing Byzantine behavior, node id: %s\n", + l.node.nodeID, ) // Byzantine behavior: Send different votes to different nodes @@ -898,7 +1022,7 @@ func (l *NodeTransitionListener) handleEnterVoting() { } // Create conflicting vote - vote := &Vote{ + vt := &vote{ NodeID: l.node.nodeID, Round: 0, // Will be updated based on proposals VoteValue: voteValues[i], @@ -906,21 +1030,21 @@ func (l *NodeTransitionListener) handleEnterVoting() { ProposerID: targetNode, } - l.logger.Warn( - "MALICIOUS: Sending conflicting vote", - zap.String("node_id", l.node.nodeID), - zap.String("target", targetNode), - zap.String("vote", voteValues[i]), + fmt.Printf( + "MALICIOUS: Sending conflicting vote, node id: %s, target: %s, vote: %s\n", + l.node.nodeID, + targetNode, + voteValues[i], ) if i == 0 && l.node.messageBus != nil { // Make a copy to avoid sharing pointers - voteCopy := &Vote{ - NodeID: vote.NodeID, - Round: vote.Round, - VoteValue: vote.VoteValue, - Timestamp: vote.Timestamp, - ProposerID: vote.ProposerID, + voteCopy := &vote{ + NodeID: vt.NodeID, + Round: vt.Round, + VoteValue: vt.VoteValue, + Timestamp: vt.Timestamp, + ProposerID: vt.ProposerID, } l.node.messageBus.Broadcast(Message{ Type: "vote", @@ -932,7 +1056,7 @@ func (l *NodeTransitionListener) handleEnterVoting() { // Also try to vote multiple times with same value time.Sleep(100 * time.Millisecond) - doubleVote := &Vote{ + doubleVote := &vote{ NodeID: l.node.nodeID, Round: 0, VoteValue: "approve", @@ -940,20 +1064,20 @@ func (l *NodeTransitionListener) handleEnterVoting() { ProposerID: nodes[0], } - l.logger.Warn( - "MALICIOUS: Attempting double vote", - zap.String("node_id", l.node.nodeID), + fmt.Printf( + "MALICIOUS: Attempting double vote, node id: %s\n", + l.node.nodeID, ) l.node.sm.ReceiveVote( - PeerID{ID: nodes[0]}, - PeerID{ID: l.node.nodeID}, + peerID{ID: nodes[0]}, + peerID{ID: l.node.nodeID}, doubleVote, ) if l.node.messageBus != nil { // Make a copy to avoid sharing pointers - doubleVoteCopy := &Vote{ + doubleVoteCopy := &vote{ NodeID: doubleVote.NodeID, Round: doubleVote.Round, VoteValue: doubleVote.VoteValue, @@ -970,21 +1094,27 @@ func (l *NodeTransitionListener) handleEnterVoting() { return } - l.logger.Info("entering voting state", - zap.String("node_id", l.node.nodeID)) + fmt.Printf( + "entering voting state, node id: %s\n", + l.node.nodeID, + ) } func (l *NodeTransitionListener) handleEnterCollecting() { - l.logger.Info("entered collecting state", - zap.String("node_id", l.node.nodeID)) + fmt.Printf( + "entered collecting state, node id: %s\n", + l.node.nodeID, + ) // Reset vote handler for new round l.node.votingProvider.Reset() } func (l *NodeTransitionListener) handleEnterPublishing() { - l.logger.Info("entered publishing state", - zap.String("node_id", l.node.nodeID)) + fmt.Printf( + "entered publishing state, node id: %s\n", + l.node.nodeID, + ) } // MessageBus simulates network communication @@ -1029,9 +1159,6 @@ func (mb *MessageBus) Broadcast(msg Message) { } func main() { - logger, _ := zap.NewDevelopment() - defer logger.Sync() - // Create message bus messageBus := NewMessageBus() @@ -1039,10 +1166,10 @@ func main() { // Note: We need 4 nodes total with vote target of 3 to demonstrate Byzantine // fault tolerance nodes := []*ConsensusNode{ - NewConsensusNode("prover-node-1", true, 3, logger.Named("prover")), - NewConsensusNode("validator-node-1", true, 3, logger.Named("validator1")), - NewConsensusNode("validator-node-2", true, 3, logger.Named("validator2")), - NewMaliciousNode("validator-node-3", false, 3, logger.Named("malicious")), + NewConsensusNode("prover-node-1", true, 3), + NewConsensusNode("validator-node-1", true, 3), + NewConsensusNode("validator-node-2", true, 3), + NewMaliciousNode("validator-node-3", false, 3), } // Connect nodes to message bus @@ -1051,15 +1178,17 @@ func main() { } // Start all nodes - logger.Info("=== Starting Consensus Network with Generic State Machine ===") - logger.Info("Using the generic state machine from consensus package") - logger.Warn("Network includes 1 MALICIOUS node (validator-node-3) demonstrating Byzantine behavior") + fmt.Println("=== Starting Consensus Network with Generic State Machine ===") + fmt.Println("Using the generic state machine from consensus package") + fmt.Println("Network includes 1 MALICIOUS node (validator-node-3) demonstrating Byzantine behavior") for _, node := range nodes { if err := node.Start(); err != nil { - logger.Fatal("failed to start node", - zap.String("node_id", node.nodeID), - zap.Error(err)) + fmt.Printf( + "failed to start node, node id: %s, %v\n", + node.nodeID, + err, + ) } } @@ -1067,34 +1196,38 @@ func main() { time.Sleep(30 * time.Second) // Print statistics - logger.Info("=== Node Statistics ===") + fmt.Println("=== Node Statistics ===") for _, node := range nodes { viz := consensus.NewStateMachineViz(node.sm) - logger.Info(fmt.Sprintf("\nStats for %s:\n%s", + fmt.Printf("\nStats for %s:\n%s", node.nodeID, - viz.GetStateStats())) + viz.GetStateStats()) - logger.Info("final state", - zap.String("node_id", node.nodeID), - zap.String("current_state", string(node.sm.GetState())), - zap.Uint64("transition_count", node.sm.GetTransitionCount()), - zap.Bool("is_malicious", node.isMalicious)) + fmt.Printf( + "final state, node id: %s, current state: %s, transition count: %s, malicious: %v\n", + node.nodeID, + string(node.sm.GetState()), + node.sm.GetTransitionCount(), + node.isMalicious, + ) } // Generate visualization if len(nodes) > 0 { viz := consensus.NewStateMachineViz(nodes[0].sm) - logger.Info("\nState Machine Diagram:\n" + viz.GenerateMermaidDiagram()) + fmt.Println("\nState Machine Diagram:\n" + viz.GenerateMermaidDiagram()) } // Stop all nodes - logger.Info("=== Stopping Consensus Network ===") + fmt.Println("=== Stopping Consensus Network ===") for _, node := range nodes { if err := node.Stop(); err != nil { - logger.Error("failed to stop node", - zap.String("node_id", node.nodeID), - zap.Error(err)) + fmt.Printf( + "failed to stop node, node id: %s, %v\n", + node.nodeID, + err, + ) } } diff --git a/consensus/forest/leveled_forest.go b/consensus/forest/leveled_forest.go index 05c726a..1ca20d3 100644 --- a/consensus/forest/leveled_forest.go +++ b/consensus/forest/leveled_forest.go @@ -229,8 +229,8 @@ func (f *LevelledForest) registerWithParent(vertexContainer *vertexContainer) { return } - _, parentView := vertexContainer.vertex.Parent() - if parentView < f.LowestLevel { + _, parentRank := vertexContainer.vertex.Parent() + if parentRank < f.LowestLevel { return } parentContainer := f.getOrCreateVertexContainer( diff --git a/consensus/forest/vertex.go b/consensus/forest/vertex.go index feccb90..a4eee35 100644 --- a/consensus/forest/vertex.go +++ b/consensus/forest/vertex.go @@ -20,7 +20,7 @@ type Vertex interface { func VertexToString(v Vertex) string { parentID, parentLevel := v.Parent() return fmt.Sprintf( - "", + "", v.VertexID(), v.Level(), parentID, diff --git a/consensus/forks/forks.go b/consensus/forks/forks.go index 958758c..be6d05f 100644 --- a/consensus/forks/forks.go +++ b/consensus/forks/forks.go @@ -354,7 +354,7 @@ func (f *Forks[StateT, VoteT]) AddValidatedState( err, ) } - err = f.checkForAdvancingFinalization(&certifiedParent) + err = f.checkForAdvancingFinalization(certifiedParent) if err != nil { return fmt.Errorf("updating finalization failed: %w", err) } @@ -650,39 +650,3 @@ func (f *Forks[StateT, VoteT]) collectStatesForFinalization( return statesToBeFinalized, nil } - -// Type used to satisfy generic arguments in compiler time type assertion check -type nilUnique struct{} - -// GetSignature implements models.Unique. -func (n *nilUnique) GetSignature() []byte { - panic("unimplemented") -} - -// GetTimestamp implements models.Unique. -func (n *nilUnique) GetTimestamp() uint64 { - panic("unimplemented") -} - -// Source implements models.Unique. -func (n *nilUnique) Source() models.Identity { - panic("unimplemented") -} - -// Clone implements models.Unique. -func (n *nilUnique) Clone() models.Unique { - panic("unimplemented") -} - -// GetRank implements models.Unique. -func (n *nilUnique) GetRank() uint64 { - panic("unimplemented") -} - -// Identity implements models.Unique. -func (n *nilUnique) Identity() models.Identity { - panic("unimplemented") -} - -var _ models.Unique = (*nilUnique)(nil) - diff --git a/consensus/forks/forks_test.go b/consensus/forks/forks_test.go new file mode 100644 index 0000000..8380307 --- /dev/null +++ b/consensus/forks/forks_test.go @@ -0,0 +1,950 @@ +package forks + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +/***************************************************************************** + * NOTATION: * + * A state is denoted as [â—„() ]. * + * For example, [â—„(1) 2] means: a state of rank 2 that has a QC for rank 1. * + *****************************************************************************/ + +// TestInitialization verifies that at initialization, Forks reports: +// - the root / genesis state as finalized +// - it has no finalization proof for the root / genesis state (state and its finalization is trusted) +func TestInitialization(t *testing.T) { + forks, _ := newForks(t) + requireOnlyGenesisStateFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) +} + +// TestFinalize_Direct1Chain tests adding a direct 1-chain on top of the genesis state: +// - receives [â—„(1) 2] [â—„(2) 5] +// +// Expected behaviour: +// - On the one hand, Forks should not finalize any _additional_ states, because there is +// no finalizable 2-chain for [â—„(1) 2]. Hence, finalization no events should be emitted. +// - On the other hand, after adding the two states, Forks has enough knowledge to construct +// a FinalityProof for the genesis state. +func TestFinalize_Direct1Chain(t *testing.T) { + builder := NewStateBuilder(). + Add(1, 2). + Add(2, 3) + states, err := builder.States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + + // adding state [â—„(1) 2] should not finalize anything + // as the genesis state is trusted, there should be no FinalityProof available for it + require.NoError(t, forks.AddValidatedState(states[0])) + requireOnlyGenesisStateFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + + // After adding state [â—„(2) 3], Forks has enough knowledge to construct a FinalityProof for the + // genesis state. However, finalization remains at the genesis state, so no events should be emitted. + expectedFinalityProof := makeFinalityProof(t, builder.GenesisState().State, states[0], states[1].ParentQuorumCertificate) + require.NoError(t, forks.AddValidatedState(states[1])) + requireLatestFinalizedState(t, forks, builder.GenesisState().State) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + + // After adding CertifiedState [â—„(1) 2] â—„(2), Forks has enough knowledge to construct a FinalityProof for + // the genesis state. However, finalization remains at the genesis state, so no events should be emitted. + expectedFinalityProof := makeFinalityProof(t, builder.GenesisState().State, states[0], states[1].ParentQuorumCertificate) + c, err := models.NewCertifiedState(states[0], states[1].ParentQuorumCertificate) + require.NoError(t, err) + + require.NoError(t, forks.AddCertifiedState(c)) + requireLatestFinalizedState(t, forks, builder.GenesisState().State) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_Direct2Chain tests adding a direct 1-chain on a direct 1-chain (direct 2-chain). +// - receives [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] +// - Forks should finalize [â—„(1) 2] +func TestFinalize_Direct2Chain(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[0], states[1], states[2].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_DirectIndirect2Chain tests adding an indirect 1-chain on a direct 1-chain. +// receives [â—„(1) 2] [â—„(2) 3] [â—„(3) 5] +// it should finalize [â—„(1) 2] +func TestFinalize_DirectIndirect2Chain(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 5). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[0], states[1], states[2].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_IndirectDirect2Chain tests adding a direct 1-chain on an indirect 1-chain. +// - Forks receives [â—„(1) 3] [â—„(3) 5] [â—„(7) 7] +// - it should not finalize any states because there is no finalizable 2-chain. +func TestFinalize_IndirectDirect2Chain(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 3). + Add(3, 5). + Add(5, 7). + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireOnlyGenesisStateFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireOnlyGenesisStateFinalized(t, forks) + _, hasProof := forks.FinalityProof() + require.False(t, hasProof) + }) +} + +// TestFinalize_Direct2ChainOnIndirect tests adding a direct 2-chain on an indirect 2-chain: +// - ingesting [â—„(1) 3] [â—„(3) 5] [â—„(5) 6] [â—„(6) 7] [â—„(7) 8] +// - should result in finalization of [â—„(5) 6] +func TestFinalize_Direct2ChainOnIndirect(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 3). + Add(3, 5). + Add(5, 6). + Add(6, 7). + Add(7, 8). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[2], states[3], states[4].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_Direct2ChainOnDirect tests adding a sequence of direct 2-chains: +// - ingesting [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] [â—„(4) 5] [â—„(5) 6] +// - should result in finalization of [â—„(3) 4] +func TestFinalize_Direct2ChainOnDirect(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + Add(4, 5). + Add(5, 6). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[2], states[3], states[4].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_Multiple2Chains tests the case where a state can be finalized by different 2-chains. +// - ingesting [â—„(1) 2] [â—„(2) 3] [â—„(3) 5] [â—„(3) 6] [â—„(3) 7] +// - should result in finalization of [â—„(1) 2] +func TestFinalize_Multiple2Chains(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 5). + Add(3, 6). + Add(3, 7). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[0], states[1], states[2].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestFinalize_OrphanedFork tests that we can finalize a state which causes a conflicting fork to be orphaned. +// We ingest the following state tree: +// +// [â—„(1) 2] [â—„(2) 3] +// [â—„(2) 4] [â—„(4) 5] [â—„(5) 6] +// +// which should result in finalization of [â—„(2) 4] and pruning of [â—„(2) 3] +func TestFinalize_OrphanedFork(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + Add(2, 3). // [â—„(2) 3], should eventually be pruned + Add(2, 4). // [â—„(2) 4], should eventually be finalized + Add(4, 5). // [â—„(4) 5] + Add(5, 6). // [â—„(5) 6] + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[2], states[3], states[4].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + require.False(t, forks.IsKnownState(states[1].Identifier)) + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + require.False(t, forks.IsKnownState(states[1].Identifier)) + requireLatestFinalizedState(t, forks, states[2]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestDuplication tests that delivering the same state/qc multiple times has +// the same end state as delivering the state/qc once. +// - Forks receives [â—„(1) 2] [â—„(2) 3] [â—„(2) 3] [â—„(3) 4] [â—„(3) 4] [â—„(4) 5] [â—„(4) 5] +// - it should finalize [â—„(2) 3] +func TestDuplication(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(2, 3). + Add(3, 4). + Add(3, 4). + Add(4, 5). + Add(4, 5). + States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[1], states[3], states[5].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[1]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states)) + + requireLatestFinalizedState(t, forks, states[1]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestIgnoreStatesBelowFinalizedRank tests that states below finalized rank are ignored. +// - Forks receives [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] [â—„(1) 5] +// - it should finalize [â—„(1) 2] +func TestIgnoreStatesBelowFinalizedRank(t *testing.T) { + builder := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + Add(2, 3). // [â—„(2) 3] + Add(3, 4). // [â—„(3) 4] + Add(1, 5) // [â—„(1) 5] + states, err := builder.States() + require.NoError(t, err) + expectedFinalityProof := makeFinalityProof(t, states[0], states[1], states[2].ParentQuorumCertificate) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + // initialize forks and add first 3 states: + // * state [â—„(1) 2] should then be finalized + // * and state [1] should be pruned + forks, _ := newForks(t) + require.Nil(t, addValidatedStateToForks(forks, states[:3])) + + // sanity checks to confirm correct test setup + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + require.False(t, forks.IsKnownState(builder.GenesisState().Identifier())) + + // adding state [â—„(1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent + // * Forks should store state, despite the parent already being pruned + // * finalization should not change + orphanedState := states[3] + require.Nil(t, forks.AddValidatedState(orphanedState)) + require.True(t, forks.IsKnownState(orphanedState.Identifier)) + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + // initialize forks and add first 3 states: + // * state [â—„(1) 2] should then be finalized + // * and state [1] should be pruned + forks, _ := newForks(t) + require.Nil(t, addCertifiedStatesToForks(forks, states[:3])) + // sanity checks to confirm correct test setup + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + require.False(t, forks.IsKnownState(builder.GenesisState().Identifier())) + + // adding state [â—„(1) 5]: note that QC is _below_ the pruning threshold, i.e. cannot resolve the parent + // * Forks should store state, despite the parent already being pruned + // * finalization should not change + certStateWithUnknownParent := toCertifiedState(t, states[3]) + require.Nil(t, forks.AddCertifiedState(certStateWithUnknownParent)) + require.True(t, forks.IsKnownState(certStateWithUnknownParent.State.Identifier)) + requireLatestFinalizedState(t, forks, states[0]) + requireFinalityProof(t, forks, expectedFinalityProof) + }) +} + +// TestDoubleProposal tests that the DoubleProposal notification is emitted when two different +// states for the same rank are added. We ingest the following state tree: +// +// / [â—„(1) 2] +// [1] +// \ [â—„(1) 2'] +// +// which should result in a DoubleProposal event referencing the states [â—„(1) 2] and [â—„(1) 2'] +func TestDoubleProposal(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + AddVersioned(1, 2, 0, 1). // [â—„(1) 2'] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[1], states[0]).Once() + + err = addValidatedStateToForks(forks, states) + require.NoError(t, err) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[1], states[0]).Once() + + err = forks.AddCertifiedState(toCertifiedState(t, states[0])) // add [â—„(1) 2] as certified state + require.NoError(t, err) + err = forks.AddCertifiedState(toCertifiedState(t, states[1])) // add [â—„(1) 2'] as certified state + require.NoError(t, err) + }) +} + +// TestConflictingQCs checks that adding 2 conflicting QCs should return models.ByzantineThresholdExceededError +// We ingest the following state tree: +// +// [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] [â—„(4) 6] +// [â—„(2) 3'] [â—„(3') 5] +// +// which should result in a `ByzantineThresholdExceededError`, because conflicting states 3 and 3' both have QCs +func TestConflictingQCs(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + Add(2, 3). // [â—„(2) 3] + AddVersioned(2, 3, 0, 1). // [â—„(2) 3'] + Add(3, 4). // [â—„(3) 4] + Add(4, 6). // [â—„(4) 6] + AddVersioned(3, 5, 1, 0). // [â—„(3') 5] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[2], states[1]).Return(nil) + + err = addValidatedStateToForks(forks, states) + assert.True(t, models.IsByzantineThresholdExceededError(err)) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[2], states[1]).Return(nil) + + // As [â—„(3') 5] is not certified, it will not be added to Forks. However, its QC â—„(3') is + // delivered to Forks as part of the *certified* state [â—„(2) 3']. + err = addCertifiedStatesToForks(forks, states) + assert.True(t, models.IsByzantineThresholdExceededError(err)) + }) +} + +// TestConflictingFinalizedForks checks that finalizing 2 conflicting forks should return models.ByzantineThresholdExceededError +// We ingest the following state tree: +// +// [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] [â—„(4) 5] +// [â—„(2) 6] [â—„(6) 7] [â—„(7) 8] +// +// Here, both states [â—„(2) 3] and [â—„(2) 6] satisfy the finalization condition, i.e. we have a fork +// in the finalized states, which should result in a models.ByzantineThresholdExceededError exception. +func TestConflictingFinalizedForks(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4). + Add(4, 5). // finalizes [â—„(2) 3] + Add(2, 6). + Add(6, 7). + Add(7, 8). // finalizes [â—„(2) 6], conflicting with conflicts with [â—„(2) 3] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + err = addValidatedStateToForks(forks, states) + assert.True(t, models.IsByzantineThresholdExceededError(err)) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + err = addCertifiedStatesToForks(forks, states) + assert.True(t, models.IsByzantineThresholdExceededError(err)) + }) +} + +// TestAddDisconnectedState checks that adding a state which does not connect to the +// latest finalized state returns a `models.MissingStateError` +// - receives [â—„(2) 3] +// - should return `models.MissingStateError`, because the parent is above the pruning +// threshold, but Forks does not know its parent +func TestAddDisconnectedState(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // we will skip this state [â—„(1) 2] + Add(2, 3). // [â—„(2) 3] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, _ := newForks(t) + err := forks.AddValidatedState(states[1]) + require.Error(t, err) + assert.True(t, models.IsMissingStateError(err)) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, _ := newForks(t) + err := forks.AddCertifiedState(toCertifiedState(t, states[1])) + require.Error(t, err) + assert.True(t, models.IsMissingStateError(err)) + }) +} + +// TestGetState tests that we can retrieve stored states. Here, we test that +// attempting to retrieve nonexistent or pruned states fails without causing an exception. +// - Forks receives [â—„(1) 2] [â—„(2) 3] [â—„(3) 4], then [â—„(4) 5] +// - should finalize [â—„(1) 2], then [â—„(2) 3] +func TestGetState(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + Add(2, 3). // [â—„(2) 3] + Add(3, 4). // [â—„(3) 4] + Add(4, 5). // [â—„(4) 5] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + statesAddedFirst := states[:3] // [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] + remainingState := states[3] // [â—„(4) 5] + forks, _ := newForks(t) + + // should be unable to retrieve a state before it is added + _, ok := forks.GetState(states[0].Identifier) + assert.False(t, ok) + + // add first 3 states - should finalize [â—„(1) 2] + err = addValidatedStateToForks(forks, statesAddedFirst) + require.NoError(t, err) + + // should be able to retrieve all stored states + for _, state := range statesAddedFirst { + b, ok := forks.GetState(state.Identifier) + assert.True(t, ok) + assert.Equal(t, state, b) + } + + // add remaining state [â—„(4) 5] - should finalize [â—„(2) 3] and prune [â—„(1) 2] + require.Nil(t, forks.AddValidatedState(remainingState)) + + // should be able to retrieve just added state + b, ok := forks.GetState(remainingState.Identifier) + assert.True(t, ok) + assert.Equal(t, remainingState, b) + + // should be unable to retrieve pruned state + _, ok = forks.GetState(statesAddedFirst[0].Identifier) + assert.False(t, ok) + }) + + // Caution: finalization is driven by QCs. Therefore, we include the QC for state 3 + // in the first batch of states that we add. This is analogous to previous test case, + // except that we are delivering the QC â—„(3) as part of the certified state of rank 2 + // [â—„(2) 3] â—„(3) + // while in the previous sub-test, the QC â—„(3) was delivered as part of state [â—„(3) 4] + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + statesAddedFirst := toCertifiedStates(t, states[:2]...) // [â—„(1) 2] [â—„(2) 3] â—„(3) + remainingState := toCertifiedState(t, states[2]) // [â—„(3) 4] â—„(4) + forks, _ := newForks(t) + + // should be unable to retrieve a state before it is added + _, ok := forks.GetState(states[0].Identifier) + assert.False(t, ok) + + // add first states - should finalize [â—„(1) 2] + err := forks.AddCertifiedState(statesAddedFirst[0]) + require.NoError(t, err) + err = forks.AddCertifiedState(statesAddedFirst[1]) + require.NoError(t, err) + + // should be able to retrieve all stored states + for _, state := range statesAddedFirst { + b, ok := forks.GetState(state.State.Identifier) + assert.True(t, ok) + assert.Equal(t, state.State, b) + } + + // add remaining state [â—„(4) 5] - should finalize [â—„(2) 3] and prune [â—„(1) 2] + require.Nil(t, forks.AddCertifiedState(remainingState)) + + // should be able to retrieve just added state + b, ok := forks.GetState(remainingState.State.Identifier) + assert.True(t, ok) + assert.Equal(t, remainingState.State, b) + + // should be unable to retrieve pruned state + _, ok = forks.GetState(statesAddedFirst[0].State.Identifier) + assert.False(t, ok) + }) +} + +// TestGetStatesForRank tests retrieving states for a rank (also including double proposals). +// - Forks receives [â—„(1) 2] [â—„(2) 4] [â—„(2) 4'], +// where [â—„(2) 4'] is a double proposal, because it has the same rank as [â—„(2) 4] +// +// Expected behaviour: +// - Forks should store all the states +// - Forks should emit a `OnDoubleProposeDetected` notification +// - we can retrieve all states, including the double proposals +func TestGetStatesForRank(t *testing.T) { + states, err := NewStateBuilder(). + Add(1, 2). // [â—„(1) 2] + Add(2, 4). // [â—„(2) 4] + AddVersioned(2, 4, 0, 1). // [â—„(2) 4'] + States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[2], states[1]).Once() + + err = addValidatedStateToForks(forks, states) + require.NoError(t, err) + + // expect 1 state at rank 2 + storedStates := forks.GetStatesForRank(2) + assert.Len(t, storedStates, 1) + assert.Equal(t, states[0], storedStates[0]) + + // expect 2 states at rank 4 + storedStates = forks.GetStatesForRank(4) + assert.Len(t, storedStates, 2) + assert.ElementsMatch(t, states[1:], storedStates) + + // expect 0 states at rank 3 + storedStates = forks.GetStatesForRank(3) + assert.Len(t, storedStates, 0) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, notifier := newForks(t) + notifier.On("OnDoubleProposeDetected", states[2], states[1]).Once() + + err := forks.AddCertifiedState(toCertifiedState(t, states[0])) + require.NoError(t, err) + err = forks.AddCertifiedState(toCertifiedState(t, states[1])) + require.NoError(t, err) + err = forks.AddCertifiedState(toCertifiedState(t, states[2])) + require.NoError(t, err) + + // expect 1 state at rank 2 + storedStates := forks.GetStatesForRank(2) + assert.Len(t, storedStates, 1) + assert.Equal(t, states[0], storedStates[0]) + + // expect 2 states at rank 4 + storedStates = forks.GetStatesForRank(4) + assert.Len(t, storedStates, 2) + assert.ElementsMatch(t, states[1:], storedStates) + + // expect 0 states at rank 3 + storedStates = forks.GetStatesForRank(3) + assert.Len(t, storedStates, 0) + }) +} + +// TestNotifications tests that Forks emits the expected events: +// - Forks receives [â—„(1) 2] [â—„(2) 3] [â—„(3) 4] +// +// Expected Behaviour: +// - Each of the ingested states should result in an `OnStateIncorporated` notification +// - Forks should finalize [â—„(1) 2], resulting in a `MakeFinal` event and an `OnFinalizedState` event +func TestNotifications(t *testing.T) { + builder := NewStateBuilder(). + Add(1, 2). + Add(2, 3). + Add(3, 4) + states, err := builder.States() + require.NoError(t, err) + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + notifier := &mocks.Consumer[*helper.TestState, *helper.TestVote]{} + // 4 states including the genesis are incorporated + notifier.On("OnStateIncorporated", mock.Anything).Return(nil).Times(4) + notifier.On("OnFinalizedState", states[0]).Once() + finalizationCallback := mocks.NewFinalizer(t) + finalizationCallback.On("MakeFinal", states[0].Identifier).Return(nil).Once() + + forks, err := NewForks(builder.GenesisState(), finalizationCallback, notifier) + require.NoError(t, err) + require.NoError(t, addValidatedStateToForks(forks, states)) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + notifier := &mocks.Consumer[*helper.TestState, *helper.TestVote]{} + // 4 states including the genesis are incorporated + notifier.On("OnStateIncorporated", mock.Anything).Return(nil).Times(4) + notifier.On("OnFinalizedState", states[0]).Once() + finalizationCallback := mocks.NewFinalizer(t) + finalizationCallback.On("MakeFinal", states[0].Identifier).Return(nil).Once() + + forks, err := NewForks(builder.GenesisState(), finalizationCallback, notifier) + require.NoError(t, err) + require.NoError(t, addCertifiedStatesToForks(forks, states)) + }) +} + +// TestFinalizingMultipleStates tests that `OnFinalizedState` notifications are emitted in correct order +// when there are multiple states finalized by adding a _single_ state. +// - receiving [â—„(1) 3] [â—„(3) 5] [â—„(5) 7] [â—„(7) 11] [â—„(11) 12] should not finalize any states, +// because there is no 2-chain with the first chain link being a _direct_ 1-chain +// - adding [â—„(12) 22] should finalize up to state [â—„(6) 11] +// +// This test verifies the following expected properties: +// 1. Safety under reentrancy: +// While Forks is single-threaded, there is still the possibility of reentrancy. Specifically, the +// consumers of our finalization events are served by the goroutine executing Forks. It is conceivable +// that a consumer might access Forks and query the latest finalization proof. This would be legal, if +// the component supplying the goroutine to Forks also consumes the notifications. Therefore, for API +// safety, we require forks to _first update_ its `FinalityProof()` before it emits _any_ events. +// 2. For each finalized state, `finalizationCallback` event is executed _before_ `OnFinalizedState` notifications. +// 3. States are finalized in order of increasing height (without skipping any states). +func TestFinalizingMultipleStates(t *testing.T) { + builder := NewStateBuilder(). + Add(1, 3). // index 0: [â—„(1) 2] + Add(3, 5). // index 1: [â—„(2) 4] + Add(5, 7). // index 2: [â—„(4) 6] + Add(7, 11). // index 3: [â—„(6) 11] -- expected to be finalized + Add(11, 12). // index 4: [â—„(11) 12] + Add(12, 22) // index 5: [â—„(12) 22] + states, err := builder.States() + require.NoError(t, err) + + // The Finality Proof should right away point to the _latest_ finalized state. Subsequently emitting + // Finalization events for lower states is fine, because notifications are guaranteed to be + // _eventually_ arriving. I.e. consumers expect notifications / events to be potentially lagging behind. + expectedFinalityProof := makeFinalityProof(t, states[3], states[4], states[5].ParentQuorumCertificate) + + setupForksAndAssertions := func() (*Forks[*helper.TestState, *helper.TestVote], *mocks.Finalizer, *mocks.Consumer[*helper.TestState, *helper.TestVote]) { + // initialize Forks with custom event consumers so we can check order of emitted events + notifier := &mocks.Consumer[*helper.TestState, *helper.TestVote]{} + finalizationCallback := mocks.NewFinalizer(t) + notifier.On("OnStateIncorporated", mock.Anything).Return(nil) + forks, err := NewForks(builder.GenesisState(), finalizationCallback, notifier) + require.NoError(t, err) + + // expecting finalization of [â—„(1) 2] [â—„(2) 4] [â—„(4) 6] [â—„(6) 11] in this order + statesAwaitingFinalization := toStateAwaitingFinalization(states[:4]) + + finalizationCallback.On("MakeFinal", mock.Anything).Run(func(args mock.Arguments) { + requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events + + // Requirement 3: finalized in order of increasing height (without skipping any states). + expectedNextFinalizationEvents := statesAwaitingFinalization[0] + require.Equal(t, expectedNextFinalizationEvents.State.Identifier, args[0]) + + // Requirement 2: finalized state, `finalizationCallback` event is executed _before_ `OnFinalizedState` notifications. + // no duplication of events under normal operations expected + require.False(t, expectedNextFinalizationEvents.MakeFinalCalled) + require.False(t, expectedNextFinalizationEvents.OnFinalizedStateEmitted) + expectedNextFinalizationEvents.MakeFinalCalled = true + }).Return(nil).Times(4) + + notifier.On("OnFinalizedState", mock.Anything).Run(func(args mock.Arguments) { + requireFinalityProof(t, forks, expectedFinalityProof) // Requirement 1: forks should _first update_ its `FinalityProof()` before it emits _any_ events + + // Requirement 3: finalized in order of increasing height (without skipping any states). + expectedNextFinalizationEvents := statesAwaitingFinalization[0] + require.Equal(t, expectedNextFinalizationEvents.State, args[0]) + + // Requirement 2: finalized state, `finalizationCallback` event is executed _before_ `OnFinalizedState` notifications. + // no duplication of events under normal operations expected + require.True(t, expectedNextFinalizationEvents.MakeFinalCalled) + require.False(t, expectedNextFinalizationEvents.OnFinalizedStateEmitted) + expectedNextFinalizationEvents.OnFinalizedStateEmitted = true + + // At this point, `MakeFinal` and `OnFinalizedState` have both been emitted for the state, so we are done with it + statesAwaitingFinalization = statesAwaitingFinalization[1:] + }).Times(4) + + return forks, finalizationCallback, notifier + } + + t.Run("consensus participant mode: ingest validated states", func(t *testing.T) { + forks, finalizationCallback, notifier := setupForksAndAssertions() + err = addValidatedStateToForks(forks, states[:5]) // adding [â—„(1) 2] [â—„(2) 4] [â—„(4) 6] [â—„(6) 11] [â—„(11) 12] + require.NoError(t, err) + requireOnlyGenesisStateFinalized(t, forks) // finalization should still be at the genesis state + + require.NoError(t, forks.AddValidatedState(states[5])) // adding [â—„(12) 22] should trigger finalization events + requireFinalityProof(t, forks, expectedFinalityProof) + finalizationCallback.AssertExpectations(t) + notifier.AssertExpectations(t) + }) + + t.Run("consensus follower mode: ingest certified states", func(t *testing.T) { + forks, finalizationCallback, notifier := setupForksAndAssertions() + // adding [â—„(1) 2] [â—„(2) 4] [â—„(4) 6] [â—„(6) 11] â—„(11) + require.NoError(t, forks.AddCertifiedState(toCertifiedState(t, states[0]))) + require.NoError(t, forks.AddCertifiedState(toCertifiedState(t, states[1]))) + require.NoError(t, forks.AddCertifiedState(toCertifiedState(t, states[2]))) + require.NoError(t, forks.AddCertifiedState(toCertifiedState(t, states[3]))) + require.NoError(t, err) + requireOnlyGenesisStateFinalized(t, forks) // finalization should still be at the genesis state + + // adding certified state [â—„(11) 12] â—„(12) should trigger finalization events + require.NoError(t, forks.AddCertifiedState(toCertifiedState(t, states[4]))) + requireFinalityProof(t, forks, expectedFinalityProof) + finalizationCallback.AssertExpectations(t) + notifier.AssertExpectations(t) + }) +} + +//* ************************************* internal functions ************************************* */ + +func newForks(t *testing.T) (*Forks[*helper.TestState, *helper.TestVote], *mocks.Consumer[*helper.TestState, *helper.TestVote]) { + notifier := mocks.NewConsumer[*helper.TestState, *helper.TestVote](t) + notifier.On("OnStateIncorporated", mock.Anything).Return(nil).Maybe() + notifier.On("OnFinalizedState", mock.Anything).Maybe() + finalizationCallback := mocks.NewFinalizer(t) + finalizationCallback.On("MakeFinal", mock.Anything).Return(nil).Maybe() + + genesisBQ := makeGenesis() + + forks, err := NewForks(genesisBQ, finalizationCallback, notifier) + + require.NoError(t, err) + return forks, notifier +} + +// addValidatedStateToForks adds all the given states to Forks, in order. +// If any errors occur, returns the first one. +func addValidatedStateToForks(forks *Forks[*helper.TestState, *helper.TestVote], states []*models.State[*helper.TestState]) error { + for _, state := range states { + err := forks.AddValidatedState(state) + if err != nil { + return fmt.Errorf("test failed to add state for rank %d: %w", state.Rank, err) + } + } + return nil +} + +// addCertifiedStatesToForks iterates over all states, caches them locally in a map, +// constructs certified states whenever possible and adds the certified states to forks, +// Note: if states is a single fork, the _last state_ in the slice will not be added, +// +// because there is no qc for it +// +// If any errors occur, returns the first one. +func addCertifiedStatesToForks(forks *Forks[*helper.TestState, *helper.TestVote], states []*models.State[*helper.TestState]) error { + uncertifiedStates := make(map[models.Identity]*models.State[*helper.TestState]) + for _, b := range states { + uncertifiedStates[b.Identifier] = b + parentID := b.ParentQuorumCertificate.GetSelector() + parent, found := uncertifiedStates[parentID] + if !found { + continue + } + delete(uncertifiedStates, parentID) + + certParent, err := models.NewCertifiedState(parent, b.ParentQuorumCertificate) + if err != nil { + return fmt.Errorf("test failed to creat certified state for rank %d: %w", certParent.State.Rank, err) + } + err = forks.AddCertifiedState(certParent) + if err != nil { + return fmt.Errorf("test failed to add certified state for rank %d: %w", certParent.State.Rank, err) + } + } + + return nil +} + +// requireLatestFinalizedState asserts that the latest finalized state has the given rank and qc rank. +func requireLatestFinalizedState(t *testing.T, forks *Forks[*helper.TestState, *helper.TestVote], expectedFinalized *models.State[*helper.TestState]) { + require.Equal(t, expectedFinalized, forks.FinalizedState(), "finalized state is not as expected") + require.Equal(t, forks.FinalizedRank(), expectedFinalized.Rank, "FinalizedRank returned wrong value") +} + +// requireOnlyGenesisStateFinalized asserts that no states have been finalized beyond the genesis state. +// Caution: does not inspect output of `forks.FinalityProof()` +func requireOnlyGenesisStateFinalized(t *testing.T, forks *Forks[*helper.TestState, *helper.TestVote]) { + genesis := makeGenesis() + require.Equal(t, forks.FinalizedState(), genesis.State, "finalized state is not the genesis state") + require.Equal(t, forks.FinalizedState().Rank, genesis.State.Rank) + require.Equal(t, forks.FinalizedState().Rank, genesis.CertifyingQuorumCertificate.GetRank()) + require.Equal(t, forks.FinalizedRank(), genesis.State.Rank, "finalized state has wrong qc") + + finalityProof, isKnown := forks.FinalityProof() + require.Nil(t, finalityProof, "expecting finality proof to be nil for genesis state at initialization") + require.False(t, isKnown, "no finality proof should be known for genesis state at initialization") +} + +// requireNoStatesFinalized asserts that no states have been finalized (genesis is latest finalized state). +func requireFinalityProof(t *testing.T, forks *Forks[*helper.TestState, *helper.TestVote], expectedFinalityProof *consensus.FinalityProof[*helper.TestState]) { + finalityProof, isKnown := forks.FinalityProof() + require.True(t, isKnown) + require.Equal(t, expectedFinalityProof, finalityProof) + require.Equal(t, forks.FinalizedState(), expectedFinalityProof.State) + require.Equal(t, forks.FinalizedRank(), expectedFinalityProof.State.Rank) +} + +// toCertifiedState generates a QC for the given state and returns their combination as a certified state +func toCertifiedState(t *testing.T, state *models.State[*helper.TestState]) *models.CertifiedState[*helper.TestState] { + qc := &helper.TestQuorumCertificate{ + Rank: state.Rank, + Selector: state.Identifier, + } + cb, err := models.NewCertifiedState(state, qc) + require.NoError(t, err) + return cb +} + +// toCertifiedStates generates a QC for the given state and returns their combination as a certified states +func toCertifiedStates(t *testing.T, states ...*models.State[*helper.TestState]) []*models.CertifiedState[*helper.TestState] { + certStates := make([]*models.CertifiedState[*helper.TestState], 0, len(states)) + for _, b := range states { + certStates = append(certStates, toCertifiedState(t, b)) + } + return certStates +} + +func makeFinalityProof(t *testing.T, state *models.State[*helper.TestState], directChild *models.State[*helper.TestState], qcCertifyingChild models.QuorumCertificate) *consensus.FinalityProof[*helper.TestState] { + c, err := models.NewCertifiedState(directChild, qcCertifyingChild) // certified child of FinalizedState + require.NoError(t, err) + return &consensus.FinalityProof[*helper.TestState]{State: state, CertifiedChild: c} +} + +// stateAwaitingFinalization is intended for tracking finalization events and their order for a specific state +type stateAwaitingFinalization struct { + State *models.State[*helper.TestState] + MakeFinalCalled bool // indicates whether `Finalizer.MakeFinal` was called + OnFinalizedStateEmitted bool // indicates whether `OnFinalizedStateCalled` notification was emitted +} + +// toStateAwaitingFinalization creates a `stateAwaitingFinalization` tracker for each input state +func toStateAwaitingFinalization(states []*models.State[*helper.TestState]) []*stateAwaitingFinalization { + trackers := make([]*stateAwaitingFinalization, 0, len(states)) + for _, b := range states { + tracker := &stateAwaitingFinalization{b, false, false} + trackers = append(trackers, tracker) + } + return trackers +} diff --git a/consensus/forks/state_builder_test.go b/consensus/forks/state_builder_test.go new file mode 100644 index 0000000..f1b0022 --- /dev/null +++ b/consensus/forks/state_builder_test.go @@ -0,0 +1,165 @@ +package forks + +import ( + "fmt" + + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +// StateRank specifies the data to create a state +type StateRank struct { + // Rank is the rank of the state to be created + Rank uint64 + // StateVersion is the version of the state for that rank. + // Useful for creating conflicting states at the same rank. + StateVersion int + // QCRank is the rank of the QC embedded in this state (also: the rank of the state's parent) + QCRank uint64 + // QCVersion is the version of the QC for that rank. + QCVersion int +} + +// QCIndex returns a unique identifier for the state's QC. +func (bv *StateRank) QCIndex() string { + return fmt.Sprintf("%v-%v", bv.QCRank, bv.QCVersion) +} + +// StateIndex returns a unique identifier for the state. +func (bv *StateRank) StateIndex() string { + return fmt.Sprintf("%v-%v", bv.Rank, bv.StateVersion) +} + +// StateBuilder is a test utility for creating state structure fixtures. +type StateBuilder struct { + stateRanks []*StateRank +} + +func NewStateBuilder() *StateBuilder { + return &StateBuilder{ + stateRanks: make([]*StateRank, 0), + } +} + +// Add adds a state with the given qcRank and stateRank. Returns self-reference for chaining. +func (bb *StateBuilder) Add(qcRank uint64, stateRank uint64) *StateBuilder { + bb.stateRanks = append(bb.stateRanks, &StateRank{ + Rank: stateRank, + QCRank: qcRank, + }) + return bb +} + +// GenesisState returns the genesis state, which is always finalized. +func (bb *StateBuilder) GenesisState() *models.CertifiedState[*helper.TestState] { + return makeGenesis() +} + +// AddVersioned adds a state with the given qcRank and stateRank. +// In addition, the version identifier of the QC embedded within the state +// is specified by `qcVersion`. The version identifier for the state itself +// (primarily for emulating different state ID) is specified by `stateVersion`. +// [(â—„3) 4] denotes a state of rank 4, with a qc for rank 3 +// [(â—„3) 4'] denotes a state of rank 4 that is different than [(â—„3) 4], with a qc for rank 3 +// [(â—„3) 4'] can be created by AddVersioned(3, 4, 0, 1) +// [(â—„3') 4] can be created by AddVersioned(3, 4, 1, 0) +// Returns self-reference for chaining. +func (bb *StateBuilder) AddVersioned(qcRank uint64, stateRank uint64, qcVersion int, stateVersion int) *StateBuilder { + bb.stateRanks = append(bb.stateRanks, &StateRank{ + Rank: stateRank, + QCRank: qcRank, + StateVersion: stateVersion, + QCVersion: qcVersion, + }) + return bb +} + +// Proposals returns a list of all proposals added to the StateBuilder. +// Returns an error if the states do not form a connected tree rooted at genesis. +func (bb *StateBuilder) Proposals() ([]*models.Proposal[*helper.TestState], error) { + states := make([]*models.Proposal[*helper.TestState], 0, len(bb.stateRanks)) + + genesisState := makeGenesis() + genesisBV := &StateRank{ + Rank: genesisState.State.Rank, + QCRank: genesisState.CertifyingQuorumCertificate.GetRank(), + } + + qcs := make(map[string]models.QuorumCertificate) + qcs[genesisBV.QCIndex()] = genesisState.CertifyingQuorumCertificate + + for _, bv := range bb.stateRanks { + qc, ok := qcs[bv.QCIndex()] + if !ok { + return nil, fmt.Errorf("test fail: no qc found for qc index: %v", bv.QCIndex()) + } + var previousRankTimeoutCert models.TimeoutCertificate + if qc.GetRank()+1 != bv.Rank { + previousRankTimeoutCert = helper.MakeTC(helper.WithTCRank(bv.Rank - 1)) + } + proposal := &models.Proposal[*helper.TestState]{ + State: &models.State[*helper.TestState]{ + Rank: bv.Rank, + ParentQuorumCertificate: qc, + }, + PreviousRankTimeoutCertificate: previousRankTimeoutCert, + } + proposal.State.Identifier = makeIdentifier(proposal.State, bv.StateVersion) + + states = append(states, proposal) + + // generate QC for the new proposal + qcs[bv.StateIndex()] = &helper.TestQuorumCertificate{ + Rank: proposal.State.Rank, + Selector: proposal.State.Identifier, + AggregatedSignature: nil, + } + } + + return states, nil +} + +// States returns a list of all states added to the StateBuilder. +// Returns an error if the states do not form a connected tree rooted at genesis. +func (bb *StateBuilder) States() ([]*models.State[*helper.TestState], error) { + proposals, err := bb.Proposals() + if err != nil { + return nil, fmt.Errorf("StateBuilder failed to generate proposals: %w", err) + } + return toStates(proposals), nil +} + +// makeIdentifier creates a state identifier based on the state's rank, QC, and state version. +// This is used to identify states uniquely, in this specific test setup. +// ATTENTION: this should not be confused with the state ID used in production code which is a collision-resistant hash +// of the full state content. +func makeIdentifier(state *models.State[*helper.TestState], stateVersion int) models.Identity { + return fmt.Sprintf("%d-%s-%d", state.Rank, state.Identifier, stateVersion) +} + +// constructs the genesis state (identical for all calls) +func makeGenesis() *models.CertifiedState[*helper.TestState] { + genesis := &models.State[*helper.TestState]{ + Rank: 1, + } + genesis.Identifier = makeIdentifier(genesis, 0) + + genesisQC := &helper.TestQuorumCertificate{ + Rank: 1, + Selector: genesis.Identifier, + } + certifiedGenesisState, err := models.NewCertifiedState(genesis, genesisQC) + if err != nil { + panic(fmt.Sprintf("combining genesis state and genensis QC to certified state failed: %s", err.Error())) + } + return certifiedGenesisState +} + +// toStates converts the given proposals to slice of states +func toStates(proposals []*models.Proposal[*helper.TestState]) []*models.State[*helper.TestState] { + states := make([]*models.State[*helper.TestState], 0, len(proposals)) + for _, b := range proposals { + states = append(states, b.State) + } + return states +} diff --git a/consensus/go.mod b/consensus/go.mod index 24ff489..21e7be1 100644 --- a/consensus/go.mod +++ b/consensus/go.mod @@ -1,9 +1,8 @@ module source.quilibrium.com/quilibrium/monorepo/consensus -go 1.23.2 - -toolchain go1.23.4 +go 1.24.0 +toolchain go1.24.9 replace github.com/multiformats/go-multiaddr => ../go-multiaddr @@ -13,36 +12,27 @@ replace github.com/libp2p/go-libp2p => ../go-libp2p replace github.com/libp2p/go-libp2p-kad-dht => ../go-libp2p-kad-dht -require go.uber.org/zap v1.27.0 +require ( + github.com/gammazero/workerpool v1.1.3 + github.com/rs/zerolog v1.34.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gammazero/deque v0.2.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + go.uber.org/goleak v1.3.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) require ( - github.com/stretchr/testify v1.10.0 - github.com/cloudflare/circl v1.6.1 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect - github.com/iden3/go-iden3-crypto v0.0.17 // indirect - github.com/ipfs/go-cid v0.5.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.1.0 // indirect - github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr v0.16.1 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.9.1 // indirect - github.com/multiformats/go-multihash v0.2.3 // indirect - github.com/multiformats/go-varint v0.0.7 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/grpc v1.72.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect - lukechampine.com/blake3 v1.4.1 // indirect github.com/pkg/errors v0.9.1 -) \ No newline at end of file + github.com/stretchr/testify v1.11.1 + go.uber.org/atomic v1.11.0 + golang.org/x/sys v0.33.0 // indirect +) diff --git a/consensus/go.sum b/consensus/go.sum index a1b6968..a5cbf1f 100644 --- a/consensus/go.sum +++ b/consensus/go.sum @@ -1,92 +1,49 @@ -github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= -github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= -github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= -github.com/iden3/go-iden3-crypto v0.0.17 h1:NdkceRLJo/pI4UpcjVah4lN/a3yzxRUGXqxbWcYh9mY= -github.com/iden3/go-iden3-crypto v0.0.17/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= -github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg= -github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= -github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= -github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= -github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.9.1 h1:x/Fuxr7ZuR4jJV4Os5g444F7xC4XmyUaT/FWtE+9Zjo= -github.com/multiformats/go-multicodec v0.9.1/go.mod h1:LLWNMtyV5ithSBUo3vFIMaeDy+h3EbkMTek1m+Fybbo= -github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= -github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= -github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA= +github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= +github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= +github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= -lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= diff --git a/consensus/helper/quorum_certificate.go b/consensus/helper/quorum_certificate.go index 76b78d8..c14bdb3 100644 --- a/consensus/helper/quorum_certificate.go +++ b/consensus/helper/quorum_certificate.go @@ -3,6 +3,7 @@ package helper import ( "bytes" crand "crypto/rand" + "fmt" "math/rand" "time" @@ -100,15 +101,15 @@ func MakeQC(options ...func(*TestQuorumCertificate)) models.QuorumCertificate { return qc } -func WithQCState[StateT models.Unique](state *models.State[StateT]) func(TestQuorumCertificate) { - return func(qc TestQuorumCertificate) { +func WithQCState[StateT models.Unique](state *models.State[StateT]) func(*TestQuorumCertificate) { + return func(qc *TestQuorumCertificate) { qc.Rank = state.Rank qc.Selector = state.Identifier } } -func WithQCSigners(signerIndices []byte) func(TestQuorumCertificate) { - return func(qc TestQuorumCertificate) { +func WithQCSigners(signerIndices []byte) func(*TestQuorumCertificate) { + return func(qc *TestQuorumCertificate) { qc.AggregatedSignature.(*TestAggregatedSignature).Bitmask = signerIndices } } @@ -116,5 +117,6 @@ func WithQCSigners(signerIndices []byte) func(TestQuorumCertificate) { func WithQCRank(rank uint64) func(*TestQuorumCertificate) { return func(qc *TestQuorumCertificate) { qc.Rank = rank + qc.Selector = fmt.Sprintf("%d", rank) } } diff --git a/consensus/helper/state.go b/consensus/helper/state.go index e2c71a9..aee6672 100644 --- a/consensus/helper/state.go +++ b/consensus/helper/state.go @@ -2,12 +2,202 @@ package helper import ( crand "crypto/rand" + "fmt" "math/rand" + "slices" "time" "source.quilibrium.com/quilibrium/monorepo/consensus/models" ) +type TestWeightedIdentity struct { + ID string +} + +// Identity implements models.WeightedIdentity. +func (t *TestWeightedIdentity) Identity() models.Identity { + return t.ID +} + +// PublicKey implements models.WeightedIdentity. +func (t *TestWeightedIdentity) PublicKey() []byte { + return make([]byte, 585) +} + +// Weight implements models.WeightedIdentity. +func (t *TestWeightedIdentity) Weight() uint64 { + return 1000 +} + +var _ models.WeightedIdentity = (*TestWeightedIdentity)(nil) + +type TestState struct { + Rank uint64 + Signature []byte + Timestamp uint64 + ID models.Identity + Prover models.Identity +} + +// Clone implements models.Unique. +func (t *TestState) Clone() models.Unique { + return &TestState{ + Rank: t.Rank, + Signature: slices.Clone(t.Signature), + Timestamp: t.Timestamp, + ID: t.ID, + Prover: t.Prover, + } +} + +// GetRank implements models.Unique. +func (t *TestState) GetRank() uint64 { + return t.Rank +} + +// GetSignature implements models.Unique. +func (t *TestState) GetSignature() []byte { + return t.Signature +} + +// GetTimestamp implements models.Unique. +func (t *TestState) GetTimestamp() uint64 { + return t.Timestamp +} + +// Identity implements models.Unique. +func (t *TestState) Identity() models.Identity { + return t.ID +} + +// Source implements models.Unique. +func (t *TestState) Source() models.Identity { + return t.Prover +} + +type TestVote struct { + Rank uint64 + Signature []byte + Timestamp uint64 + ID models.Identity + StateID models.Identity +} + +// Clone implements models.Unique. +func (t *TestVote) Clone() models.Unique { + return &TestVote{ + Rank: t.Rank, + Signature: slices.Clone(t.Signature), + Timestamp: t.Timestamp, + ID: t.ID, + StateID: t.StateID, + } +} + +// GetRank implements models.Unique. +func (t *TestVote) GetRank() uint64 { + return t.Rank +} + +// GetSignature implements models.Unique. +func (t *TestVote) GetSignature() []byte { + return t.Signature +} + +// GetTimestamp implements models.Unique. +func (t *TestVote) GetTimestamp() uint64 { + return t.Timestamp +} + +// Identity implements models.Unique. +func (t *TestVote) Identity() models.Identity { + return t.ID +} + +// Source implements models.Unique. +func (t *TestVote) Source() models.Identity { + return t.StateID +} + +type TestPeer struct { + PeerID string +} + +// Clone implements models.Unique. +func (t *TestPeer) Clone() models.Unique { + return &TestPeer{ + PeerID: t.PeerID, + } +} + +// GetRank implements models.Unique. +func (t *TestPeer) GetRank() uint64 { + return 0 +} + +// GetSignature implements models.Unique. +func (t *TestPeer) GetSignature() []byte { + return []byte{} +} + +// GetTimestamp implements models.Unique. +func (t *TestPeer) GetTimestamp() uint64 { + return 0 +} + +// Identity implements models.Unique. +func (t *TestPeer) Identity() models.Identity { + return t.PeerID +} + +// Source implements models.Unique. +func (t *TestPeer) Source() models.Identity { + return t.PeerID +} + +type TestCollected struct { + Rank uint64 + TXs [][]byte +} + +// Clone implements models.Unique. +func (t *TestCollected) Clone() models.Unique { + return &TestCollected{ + Rank: t.Rank, + TXs: slices.Clone(t.TXs), + } +} + +// GetRank implements models.Unique. +func (t *TestCollected) GetRank() uint64 { + return t.Rank +} + +// GetSignature implements models.Unique. +func (t *TestCollected) GetSignature() []byte { + return []byte{} +} + +// GetTimestamp implements models.Unique. +func (t *TestCollected) GetTimestamp() uint64 { + return 0 +} + +// Identity implements models.Unique. +func (t *TestCollected) Identity() models.Identity { + return fmt.Sprintf("%d", t.Rank) +} + +// Source implements models.Unique. +func (t *TestCollected) Source() models.Identity { + return "" +} + +var _ models.Unique = (*TestState)(nil) +var _ models.Unique = (*TestVote)(nil) +var _ models.Unique = (*TestPeer)(nil) +var _ models.Unique = (*TestCollected)(nil) + func MakeIdentity() models.Identity { s := make([]byte, 32) crand.Read(s) @@ -110,3 +300,43 @@ func WithPreviousRankTimeoutCertificate[StateT models.Unique](previousRankTimeou proposal.PreviousRankTimeoutCertificate = previousRankTimeoutCert } } + +func WithWeightedIdentityList(count int) []models.WeightedIdentity { + wi := []models.WeightedIdentity{} + for i := range count { + wi = append(wi, &TestWeightedIdentity{ + ID: fmt.Sprintf("%d", i), + }) + } + return wi +} + +func VoteForStateFixture[StateT models.Unique, VoteT models.Unique](state *models.State[StateT], ops ...func(vote *VoteT)) VoteT { + v := new(VoteT) + for _, op := range ops { + op(v) + } + return *v +} + +func VoteFixture[VoteT models.Unique](op func(vote *VoteT)) VoteT { + v := new(VoteT) + op(v) + return *v +} + +type FmtLog struct{} + +// Error implements consensus.TraceLogger. +func (n *FmtLog) Error(message string, err error) { + fmt.Printf("ERROR: %s: %v\n", message, err) +} + +// Trace implements consensus.TraceLogger. +func (n *FmtLog) Trace(message string) { + fmt.Printf("TRACE: %s\n", message) +} + +func Logger() *FmtLog { + return &FmtLog{} +} diff --git a/consensus/helper/timeout_certificate.go b/consensus/helper/timeout_certificate.go index 4f8875a..3b25192 100644 --- a/consensus/helper/timeout_certificate.go +++ b/consensus/helper/timeout_certificate.go @@ -125,9 +125,21 @@ func TimeoutStateFixture[VoteT models.Unique]( opt(timeout) } + if timeout.Vote == nil { + panic("WithTimeoutVote must be called") + } + return timeout } +func WithTimeoutVote[VoteT models.Unique]( + vote VoteT, +) func(*models.TimeoutState[VoteT]) { + return func(state *models.TimeoutState[VoteT]) { + state.Vote = &vote + } +} + func WithTimeoutNewestQC[VoteT models.Unique]( newestQC models.QuorumCertificate, ) func(*models.TimeoutState[VoteT]) { @@ -149,5 +161,11 @@ func WithTimeoutStateRank[VoteT models.Unique]( ) func(*models.TimeoutState[VoteT]) { return func(timeout *models.TimeoutState[VoteT]) { timeout.Rank = rank + if timeout.LatestQuorumCertificate != nil { + timeout.LatestQuorumCertificate.(*TestQuorumCertificate).Rank = rank + } + if timeout.PriorRankTimeoutCertificate != nil { + timeout.PriorRankTimeoutCertificate.(*TestTimeoutCertificate).Rank = rank - 1 + } } } diff --git a/consensus/mocks/dynamic_committee.go b/consensus/mocks/dynamic_committee.go index e48b6a7..468182c 100644 --- a/consensus/mocks/dynamic_committee.go +++ b/consensus/mocks/dynamic_committee.go @@ -103,23 +103,23 @@ func (_m *DynamicCommittee) IdentityByRank(rank uint64, participantID models.Ide } // IdentityByState provides a mock function with given fields: stateID, participantID -func (_m *DynamicCommittee) IdentityByState(stateID models.Identity, participantID models.Identity) (*models.WeightedIdentity, error) { +func (_m *DynamicCommittee) IdentityByState(stateID models.Identity, participantID models.Identity) (models.WeightedIdentity, error) { ret := _m.Called(stateID, participantID) if len(ret) == 0 { panic("no return value specified for IdentityByState") } - var r0 *models.WeightedIdentity + var r0 models.WeightedIdentity var r1 error - if rf, ok := ret.Get(0).(func(models.Identity, models.Identity) (*models.WeightedIdentity, error)); ok { + if rf, ok := ret.Get(0).(func(models.Identity, models.Identity) (models.WeightedIdentity, error)); ok { return rf(stateID, participantID) } - if rf, ok := ret.Get(0).(func(models.Identity, models.Identity) *models.WeightedIdentity); ok { + if rf, ok := ret.Get(0).(func(models.Identity, models.Identity) models.WeightedIdentity); ok { r0 = rf(stateID, participantID) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*models.WeightedIdentity) + r0 = ret.Get(0).(models.WeightedIdentity) } } diff --git a/consensus/mocks/signer.go b/consensus/mocks/signer.go index 618a92b..b29893c 100644 --- a/consensus/mocks/signer.go +++ b/consensus/mocks/signer.go @@ -12,9 +12,9 @@ type Signer[StateT models.Unique, VoteT models.Unique] struct { mock.Mock } -// CreateTimeout provides a mock function with given fields: curView, newestQC, previousRankTimeoutCert -func (_m *Signer[StateT, VoteT]) CreateTimeout(curView uint64, newestQC models.QuorumCertificate, previousRankTimeoutCert models.TimeoutCertificate) (*models.TimeoutState[VoteT], error) { - ret := _m.Called(curView, newestQC, previousRankTimeoutCert) +// CreateTimeout provides a mock function with given fields: curRank, newestQC, previousRankTimeoutCert +func (_m *Signer[StateT, VoteT]) CreateTimeout(curRank uint64, newestQC models.QuorumCertificate, previousRankTimeoutCert models.TimeoutCertificate) (*models.TimeoutState[VoteT], error) { + ret := _m.Called(curRank, newestQC, previousRankTimeoutCert) if len(ret) == 0 { panic("no return value specified for CreateTimeout") @@ -23,10 +23,10 @@ func (_m *Signer[StateT, VoteT]) CreateTimeout(curView uint64, newestQC models.Q var r0 *models.TimeoutState[VoteT] var r1 error if rf, ok := ret.Get(0).(func(uint64, models.QuorumCertificate, models.TimeoutCertificate) (*models.TimeoutState[VoteT], error)); ok { - return rf(curView, newestQC, previousRankTimeoutCert) + return rf(curRank, newestQC, previousRankTimeoutCert) } if rf, ok := ret.Get(0).(func(uint64, models.QuorumCertificate, models.TimeoutCertificate) *models.TimeoutState[VoteT]); ok { - r0 = rf(curView, newestQC, previousRankTimeoutCert) + r0 = rf(curRank, newestQC, previousRankTimeoutCert) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*models.TimeoutState[VoteT]) @@ -34,7 +34,7 @@ func (_m *Signer[StateT, VoteT]) CreateTimeout(curView uint64, newestQC models.Q } if rf, ok := ret.Get(1).(func(uint64, models.QuorumCertificate, models.TimeoutCertificate) error); ok { - r1 = rf(curView, newestQC, previousRankTimeoutCert) + r1 = rf(curRank, newestQC, previousRankTimeoutCert) } else { r1 = ret.Error(1) } diff --git a/consensus/mocks/verifying_vote_processor.go b/consensus/mocks/verifying_vote_processor.go index 6e34cc3..46852c1 100644 --- a/consensus/mocks/verifying_vote_processor.go +++ b/consensus/mocks/verifying_vote_processor.go @@ -33,19 +33,19 @@ func (_m *VerifyingVoteProcessor[StateT, VoteT]) Process(vote *VoteT) error { } // State provides a mock function with no fields -func (_m *VerifyingVoteProcessor[StateT, VoteT]) State() *StateT { +func (_m *VerifyingVoteProcessor[StateT, VoteT]) State() *models.State[StateT] { ret := _m.Called() if len(ret) == 0 { panic("no return value specified for State") } - var r0 *StateT - if rf, ok := ret.Get(0).(func() *StateT); ok { + var r0 *models.State[StateT] + if rf, ok := ret.Get(0).(func() *models.State[StateT]); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*StateT) + r0 = ret.Get(0).(*models.State[StateT]) } } diff --git a/consensus/mocks/vote_collector.go b/consensus/mocks/vote_collector.go index ca72d71..c5eef93 100644 --- a/consensus/mocks/vote_collector.go +++ b/consensus/mocks/vote_collector.go @@ -50,6 +50,24 @@ func (_m *VoteCollector[StateT, VoteT]) ProcessState(state *models.SignedProposa return r0 } +// Rank provides a mock function with no fields +func (_m *VoteCollector[StateT, VoteT]) Rank() uint64 { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Rank") + } + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + return r0 +} + // RegisterVoteConsumer provides a mock function with given fields: consumer func (_m *VoteCollector[StateT, VoteT]) RegisterVoteConsumer(consumer consensus.VoteConsumer[VoteT]) { _m.Called(consumer) @@ -73,24 +91,6 @@ func (_m *VoteCollector[StateT, VoteT]) Status() consensus.VoteCollectorStatus { return r0 } -// View provides a mock function with no fields -func (_m *VoteCollector[StateT, VoteT]) View() uint64 { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for View") - } - - var r0 uint64 - if rf, ok := ret.Get(0).(func() uint64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(uint64) - } - - return r0 -} - // NewVoteCollector creates a new instance of VoteCollector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewVoteCollector[StateT models.Unique, VoteT models.Unique](t interface { diff --git a/consensus/mocks/vote_processor_factory.go b/consensus/mocks/vote_processor_factory.go index c1cb62f..72b24cd 100644 --- a/consensus/mocks/vote_processor_factory.go +++ b/consensus/mocks/vote_processor_factory.go @@ -14,9 +14,9 @@ type VoteProcessorFactory[StateT models.Unique, VoteT models.Unique] struct { mock.Mock } -// Create provides a mock function with given fields: tracer, proposal -func (_m *VoteProcessorFactory[StateT, VoteT]) Create(tracer consensus.TraceLogger, proposal *models.SignedProposal[StateT, VoteT]) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) { - ret := _m.Called(tracer, proposal) +// Create provides a mock function with given fields: tracer, proposal, dsTag, aggregator +func (_m *VoteProcessorFactory[StateT, VoteT]) Create(tracer consensus.TraceLogger, proposal *models.SignedProposal[StateT, VoteT], dsTag []byte, aggregator consensus.SignatureAggregator) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) { + ret := _m.Called(tracer, proposal, dsTag, aggregator) if len(ret) == 0 { panic("no return value specified for Create") @@ -24,19 +24,19 @@ func (_m *VoteProcessorFactory[StateT, VoteT]) Create(tracer consensus.TraceLogg var r0 consensus.VerifyingVoteProcessor[StateT, VoteT] var r1 error - if rf, ok := ret.Get(0).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT]) (consensus.VerifyingVoteProcessor[StateT, VoteT], error)); ok { - return rf(tracer, proposal) + if rf, ok := ret.Get(0).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT], []byte, consensus.SignatureAggregator) (consensus.VerifyingVoteProcessor[StateT, VoteT], error)); ok { + return rf(tracer, proposal, dsTag, aggregator) } - if rf, ok := ret.Get(0).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT]) consensus.VerifyingVoteProcessor[StateT, VoteT]); ok { - r0 = rf(tracer, proposal) + if rf, ok := ret.Get(0).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT], []byte, consensus.SignatureAggregator) consensus.VerifyingVoteProcessor[StateT, VoteT]); ok { + r0 = rf(tracer, proposal, dsTag, aggregator) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(consensus.VerifyingVoteProcessor[StateT, VoteT]) } } - if rf, ok := ret.Get(1).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT]) error); ok { - r1 = rf(tracer, proposal) + if rf, ok := ret.Get(1).(func(consensus.TraceLogger, *models.SignedProposal[StateT, VoteT], []byte, consensus.SignatureAggregator) error); ok { + r1 = rf(tracer, proposal, dsTag, aggregator) } else { r1 = ret.Error(1) } diff --git a/consensus/mocks/voting_provider.go b/consensus/mocks/voting_provider.go index ababa49..4c8892f 100644 --- a/consensus/mocks/voting_provider.go +++ b/consensus/mocks/voting_provider.go @@ -14,6 +14,36 @@ type VotingProvider[StateT models.Unique, VoteT models.Unique, PeerIDT models.Un mock.Mock } +// FinalizeQuorumCertificate provides a mock function with given fields: ctx, state, aggregatedSignature +func (_m *VotingProvider[StateT, VoteT, PeerIDT]) FinalizeQuorumCertificate(ctx context.Context, state *models.State[StateT], aggregatedSignature models.AggregatedSignature) (models.QuorumCertificate, error) { + ret := _m.Called(ctx, state, aggregatedSignature) + + if len(ret) == 0 { + panic("no return value specified for FinalizeQuorumCertificate") + } + + var r0 models.QuorumCertificate + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *models.State[StateT], models.AggregatedSignature) (models.QuorumCertificate, error)); ok { + return rf(ctx, state, aggregatedSignature) + } + if rf, ok := ret.Get(0).(func(context.Context, *models.State[StateT], models.AggregatedSignature) models.QuorumCertificate); ok { + r0 = rf(ctx, state, aggregatedSignature) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(models.QuorumCertificate) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *models.State[StateT], models.AggregatedSignature) error); ok { + r1 = rf(ctx, state, aggregatedSignature) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FinalizeTimeout provides a mock function with given fields: ctx, filter, rank, latestQuorumCertificateRanks, aggregatedSignature func (_m *VotingProvider[StateT, VoteT, PeerIDT]) FinalizeTimeout(ctx context.Context, filter []byte, rank uint64, latestQuorumCertificateRanks []uint64, aggregatedSignature models.AggregatedSignature) (models.TimeoutCertificate, error) { ret := _m.Called(ctx, filter, rank, latestQuorumCertificateRanks, aggregatedSignature) diff --git a/consensus/models/state.go b/consensus/models/state.go index a823ef1..59170f6 100644 --- a/consensus/models/state.go +++ b/consensus/models/state.go @@ -65,9 +65,9 @@ type CertifiedState[StateT Unique] struct { func NewCertifiedState[StateT Unique]( state *State[StateT], quorumCertificate QuorumCertificate, -) (CertifiedState[StateT], error) { +) (*CertifiedState[StateT], error) { if state.Rank != quorumCertificate.GetRank() { - return CertifiedState[StateT]{}, + return &CertifiedState[StateT]{}, fmt.Errorf( "state's rank (%d) should equal the qc's rank (%d)", state.Rank, @@ -75,14 +75,14 @@ func NewCertifiedState[StateT Unique]( ) } if state.Identifier != quorumCertificate.GetSelector() { - return CertifiedState[StateT]{}, + return &CertifiedState[StateT]{}, fmt.Errorf( "state's ID (%x) should equal the state referenced by the qc (%x)", state.Identifier, quorumCertificate.GetSelector(), ) } - return CertifiedState[StateT]{ + return &CertifiedState[StateT]{ State: state, CertifyingQuorumCertificate: quorumCertificate, }, nil diff --git a/consensus/models/timeout_state.go b/consensus/models/timeout_state.go index c847f4b..d301e5e 100644 --- a/consensus/models/timeout_state.go +++ b/consensus/models/timeout_state.go @@ -1,6 +1,8 @@ package models -import "bytes" +import ( + "bytes" +) // TimeoutState represents the stored state change step relevant to the point of // rank of a given instance of the consensus state machine. @@ -36,10 +38,21 @@ func (t *TimeoutState[VoteT]) Equals(other *TimeoutState[VoteT]) bool { return false } + if t.Vote != other.Vote && (other.Vote == nil || t.Vote == nil) { + return false + } + // both are not nil, so we can compare the fields return t.Rank == other.Rank && - t.LatestQuorumCertificate.Equals(other.LatestQuorumCertificate) && - t.PriorRankTimeoutCertificate.Equals(other.PriorRankTimeoutCertificate) && - (*t.Vote).Source() == (*other.Vote).Source() && - bytes.Equal((*t.Vote).GetSignature(), (*other.Vote).GetSignature()) + ((t.LatestQuorumCertificate == nil && + other.LatestQuorumCertificate == nil) || + t.LatestQuorumCertificate.Equals(other.LatestQuorumCertificate)) && + ((t.PriorRankTimeoutCertificate == nil && + other.PriorRankTimeoutCertificate == nil) || + t.PriorRankTimeoutCertificate.Equals( + other.PriorRankTimeoutCertificate, + )) && + ((t.Vote == other.Vote) || + ((*t.Vote).Source() == (*other.Vote).Source()) && + bytes.Equal((*t.Vote).GetSignature(), (*other.Vote).GetSignature())) } diff --git a/consensus/notifications/pubsub/communicator_distributor.go b/consensus/notifications/pubsub/communicator_distributor.go index e125486..80047ee 100644 --- a/consensus/notifications/pubsub/communicator_distributor.go +++ b/consensus/notifications/pubsub/communicator_distributor.go @@ -10,7 +10,7 @@ import ( // CommunicatorDistributor ingests outbound consensus messages from HotStuff's // core logic and distributes them to consumers. This logic only runs inside -// active consensus participants proposing state, voting, collecting + +// active consensus participants proposing states, voting, collecting + // aggregating votes to QCs, and participating in the pacemaker (sending // timeouts, collecting + aggregating timeouts to TCs). // Concurrency safe. diff --git a/consensus/notifications/pubsub/participant_distributor.go b/consensus/notifications/pubsub/participant_distributor.go index 0a7e053..8069df5 100644 --- a/consensus/notifications/pubsub/participant_distributor.go +++ b/consensus/notifications/pubsub/participant_distributor.go @@ -52,108 +52,108 @@ func ( func ( d *ParticipantDistributor[StateT, VoteT], -) OnStart(currentView uint64) { +) OnStart(currentRank uint64) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnStart(currentView) + subscriber.OnStart(currentRank) } } func ( d *ParticipantDistributor[StateT, VoteT], ) OnReceiveProposal( - currentView uint64, + currentRank uint64, proposal *models.SignedProposal[StateT, VoteT], ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnReceiveProposal(currentView, proposal) + subscriber.OnReceiveProposal(currentRank, proposal) } } func ( d *ParticipantDistributor[StateT, VoteT], -) OnReceiveQuorumCertificate(currentView uint64, qc models.QuorumCertificate) { +) OnReceiveQuorumCertificate(currentRank uint64, qc models.QuorumCertificate) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnReceiveQuorumCertificate(currentView, qc) + subscriber.OnReceiveQuorumCertificate(currentRank, qc) } } func ( d *ParticipantDistributor[StateT, VoteT], ) OnReceiveTimeoutCertificate( - currentView uint64, + currentRank uint64, tc models.TimeoutCertificate, ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnReceiveTimeoutCertificate(currentView, tc) + subscriber.OnReceiveTimeoutCertificate(currentRank, tc) } } func ( d *ParticipantDistributor[StateT, VoteT], ) OnPartialTimeoutCertificate( - currentView uint64, - partialTc *consensus.PartialTimeoutCertificateCreated, + currentRank uint64, + partialTimeoutCertificate *consensus.PartialTimeoutCertificateCreated, ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnPartialTimeoutCertificate(currentView, partialTc) + subscriber.OnPartialTimeoutCertificate(currentRank, partialTimeoutCertificate) } } func ( d *ParticipantDistributor[StateT, VoteT], -) OnLocalTimeout(currentView uint64) { +) OnLocalTimeout(currentRank uint64) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnLocalTimeout(currentView) + subscriber.OnLocalTimeout(currentRank) } } func ( d *ParticipantDistributor[StateT, VoteT], -) OnRankChange(oldView, newView uint64) { +) OnRankChange(oldRank, newRank uint64) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnRankChange(oldView, newView) + subscriber.OnRankChange(oldRank, newRank) } } func ( d *ParticipantDistributor[StateT, VoteT], ) OnQuorumCertificateTriggeredRankChange( - oldView uint64, - newView uint64, + oldRank uint64, + newRank uint64, qc models.QuorumCertificate, ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnQuorumCertificateTriggeredRankChange(oldView, newView, qc) + subscriber.OnQuorumCertificateTriggeredRankChange(oldRank, newRank, qc) } } func ( d *ParticipantDistributor[StateT, VoteT], ) OnTimeoutCertificateTriggeredRankChange( - oldView uint64, - newView uint64, + oldRank uint64, + newRank uint64, tc models.TimeoutCertificate, ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnTimeoutCertificateTriggeredRankChange(oldView, newView, tc) + subscriber.OnTimeoutCertificateTriggeredRankChange(oldRank, newRank, tc) } } @@ -170,12 +170,12 @@ func ( func ( d *ParticipantDistributor[StateT, VoteT], ) OnCurrentRankDetails( - currentView, finalizedView uint64, + currentRank, finalizedRank uint64, currentLeader models.Identity, ) { d.lock.RLock() defer d.lock.RUnlock() for _, subscriber := range d.consumers { - subscriber.OnCurrentRankDetails(currentView, finalizedView, currentLeader) + subscriber.OnCurrentRankDetails(currentRank, finalizedRank, currentLeader) } } diff --git a/consensus/pacemaker/pacemaker.go b/consensus/pacemaker/pacemaker.go index 5ead574..4861d31 100644 --- a/consensus/pacemaker/pacemaker.go +++ b/consensus/pacemaker/pacemaker.go @@ -48,6 +48,7 @@ func NewPacemaker[ store: store, traceLogger: traceLogger, livenessState: livenessState, + backoffTimer: consensus.NewBackoffTimer(), started: false, }, nil } diff --git a/consensus/safetyrules/safety_rules.go b/consensus/safetyrules/safety_rules.go index 3fc22e2..9346276 100644 --- a/consensus/safetyrules/safety_rules.go +++ b/consensus/safetyrules/safety_rules.go @@ -207,7 +207,7 @@ func (r *SafetyRules[StateT, VoteT]) produceVote( } } - vote, err := r.signer.CreateVote(state.State) + vote, err := r.signer.CreateVote(state) if err != nil { return nil, fmt.Errorf("could not vote for state: %w", err) } @@ -223,7 +223,7 @@ func (r *SafetyRules[StateT, VoteT]) produceVote( return nil, fmt.Errorf("could not persist safety data: %w", err) } - return &vote, nil + return vote, nil } // ProduceTimeout takes current rank, highest locally known QC and TC (optional, @@ -252,6 +252,7 @@ func (r *SafetyRules[StateT, VoteT]) ProduceTimeout( LatestQuorumCertificate: lastTimeout.LatestQuorumCertificate, PriorRankTimeoutCertificate: lastTimeout.PriorRankTimeoutCertificate, TimeoutTick: lastTimeout.TimeoutTick + 1, + Vote: lastTimeout.Vote, } // persist updated TimeoutState in `safetyData` and return it diff --git a/consensus/safetyrules/safety_rules_test.go b/consensus/safetyrules/safety_rules_test.go new file mode 100644 index 0000000..f47d2de --- /dev/null +++ b/consensus/safetyrules/safety_rules_test.go @@ -0,0 +1,834 @@ +package safetyrules + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +func TestSafetyRules(t *testing.T) { + suite.Run(t, new(SafetyRulesTestSuite)) +} + +// SafetyRulesTestSuite is a test suite for testing SafetyRules related functionality. +// SafetyRulesTestSuite setups mocks for injected modules and creates models.ConsensusState[*helper.TestVote] +// based on next configuration: +// R <- B[QC_R] <- P[QC_B] +// B.Rank = S.Rank + 1 +// B - bootstrapped state, we are creating SafetyRules at state B +// Based on this LatestAcknowledgedRank = B.Rank and +type SafetyRulesTestSuite struct { + suite.Suite + + bootstrapState *models.State[*helper.TestState] + proposal *models.SignedProposal[*helper.TestState, *helper.TestVote] + proposerIdentity models.Identity + ourIdentity models.Identity + signer *mocks.Signer[*helper.TestState, *helper.TestVote] + persister *mocks.ConsensusStore[*helper.TestVote] + committee *mocks.DynamicCommittee + safetyData *models.ConsensusState[*helper.TestVote] + safety *SafetyRules[*helper.TestState, *helper.TestVote] +} + +func (s *SafetyRulesTestSuite) SetupTest() { + s.ourIdentity = helper.MakeIdentity() + s.signer = &mocks.Signer[*helper.TestState, *helper.TestVote]{} + s.persister = &mocks.ConsensusStore[*helper.TestVote]{} + s.committee = &mocks.DynamicCommittee{} + s.proposerIdentity = helper.MakeIdentity() + + // bootstrap at random bootstrapState + s.bootstrapState = helper.MakeState(helper.WithStateRank[*helper.TestState](100)) + s.proposal = helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState[*helper.TestState]( + helper.MakeState[*helper.TestState]( + helper.WithParentState[*helper.TestState](s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+1), + helper.WithStateProposer[*helper.TestState](s.proposerIdentity)), + )))) + + s.committee.On("Self").Return(s.ourIdentity).Maybe() + s.committee.On("LeaderForRank", mock.Anything).Return(s.proposerIdentity, nil).Maybe() + s.committee.On("IdentityByState", mock.Anything, s.ourIdentity).Return(&helper.TestWeightedIdentity{ID: s.ourIdentity}, nil).Maybe() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.proposal.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + s.committee.On("IdentityByRank", mock.Anything, s.ourIdentity).Return(&helper.TestWeightedIdentity{ID: s.ourIdentity}, nil).Maybe() + + s.safetyData = &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.bootstrapState.Rank, + LatestAcknowledgedRank: s.bootstrapState.Rank, + } + + s.persister.On("GetConsensusState").Return(s.safetyData, nil).Once() + var err error + s.safety, err = NewSafetyRules(s.signer, s.persister, s.committee) + require.NoError(s.T(), err) +} + +// TestProduceVote_ShouldVote test basic happy path scenario where we vote for first state after bootstrap +// and next rank ended with TC +func (s *SafetyRulesTestSuite) TestProduceVote_ShouldVote() { + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.proposal.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: s.proposal.State.Rank, + } + + expectedVote := makeVote(s.proposal.State) + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), &expectedVote, vote) + + s.persister.AssertCalled(s.T(), "PutConsensusState", expectedSafetyData) + + // producing vote for same rank yields an error since we have voted already for this rank + otherVote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), otherVote) + + previousRankTimeoutCert := helper.MakeTC( + helper.WithTCRank(s.proposal.State.Rank+1), + helper.WithTCNewestQC(s.proposal.State.ParentQuorumCertificate)) + + // voting on proposal where last rank ended with TC + proposalWithTC := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal[*helper.TestState]( + helper.WithState[*helper.TestState]( + helper.MakeState[*helper.TestState]( + helper.WithParentState[*helper.TestState](s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.proposal.State.Rank+2), + helper.WithStateProposer[*helper.TestState](s.proposerIdentity))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState](previousRankTimeoutCert)))) + + expectedSafetyData = &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.proposal.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: proposalWithTC.State.Rank, + } + + expectedVote = makeVote(proposalWithTC.State) + s.signer.On("CreateVote", proposalWithTC.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + s.committee.On("IdentityByState", proposalWithTC.State.Identifier, proposalWithTC.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + + vote, err = s.safety.ProduceVote(proposalWithTC, proposalWithTC.State.Rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), &expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutConsensusState", expectedSafetyData) +} + +// TestProduceVote_IncludedQCHigherThanTCsQC checks specific scenario where previous round resulted in TC and leader +// knows about QC which is not part of TC and qc.Rank > tc.NewestQC.Rank. We want to allow this, in this case leader +// includes their QC into proposal satisfies next condition: State.ParentQuorumCertificate.GetRank() > previousRankTimeoutCert.NewestQC.Rank +func (s *SafetyRulesTestSuite) TestProduceVote_IncludedQCHigherThanTCsQC() { + previousRankTimeoutCert := helper.MakeTC( + helper.WithTCRank(s.proposal.State.Rank+1), + helper.WithTCNewestQC(s.proposal.State.ParentQuorumCertificate)) + + // voting on proposal where last rank ended with TC + proposalWithTC := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal[*helper.TestState]( + helper.WithState[*helper.TestState]( + helper.MakeState[*helper.TestState]( + helper.WithParentState[*helper.TestState](s.proposal.State), + helper.WithStateRank[*helper.TestState](s.proposal.State.Rank+2), + helper.WithStateProposer[*helper.TestState](s.proposerIdentity))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState](previousRankTimeoutCert)))) + + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: proposalWithTC.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: proposalWithTC.State.Rank, + } + + require.Greater(s.T(), proposalWithTC.State.ParentQuorumCertificate.GetRank(), proposalWithTC.PreviousRankTimeoutCertificate.GetLatestQuorumCert().GetRank(), + "for this test case we specifically require that qc.Rank > previousRankTimeoutCert.NewestQC.Rank") + + expectedVote := makeVote(proposalWithTC.State) + s.signer.On("CreateVote", proposalWithTC.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + s.committee.On("IdentityByState", proposalWithTC.State.Identifier, proposalWithTC.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + + vote, err := s.safety.ProduceVote(proposalWithTC, proposalWithTC.State.Rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), &expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutConsensusState", expectedSafetyData) +} + +// TestProduceVote_UpdateFinalizedRank tests that FinalizedRank is updated when sees a higher QC. +// Note: `FinalizedRank` is only updated when the replica votes. +func (s *SafetyRulesTestSuite) TestProduceVote_UpdateFinalizedRank() { + s.safety.consensusState.FinalizedRank = 0 + + require.NotEqual(s.T(), s.safety.consensusState.FinalizedRank, s.proposal.State.ParentQuorumCertificate.GetRank(), + "in this test FinalizedRank is lower so it needs to be updated") + + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.proposal.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: s.proposal.State.Rank, + } + + expectedVote := makeVote(s.proposal.State) + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), &expectedVote, vote) + s.signer.AssertExpectations(s.T()) + s.persister.AssertCalled(s.T(), "PutConsensusState", expectedSafetyData) +} + +// TestProduceVote_InvalidCurrentRank tests that no vote is created if `curRank` has invalid values. +// In particular, `SafetyRules` requires that: +// - the state's rank matches `curRank` +// - that values for `curRank` are monotonicly increasing +// +// Failing any of these conditions is a symptom of an internal bug; hence `SafetyRules` should +// _not_ return a `NoVoteError`. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidCurrentRank() { + + s.Run("state-rank-does-not-match", func() { + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank+1) + require.Nil(s.T(), vote) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + }) + s.Run("rank-not-monotonicly-increasing", func() { + // create state with rank < LatestAcknowledgedRank + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + func(state *models.State[*helper.TestState]) { + state.ParentQuorumCertificate = helper.MakeQC(helper.WithQCRank(s.safetyData.LatestAcknowledgedRank - 2)) + }, + helper.WithStateRank[*helper.TestState](s.safetyData.LatestAcknowledgedRank-1)))))) + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Nil(s.T(), vote) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + }) + + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_CommitteeLeaderException verifies that SafetyRules handles unexpected error returns from +// the DynamicCommittee correctly. Specifically, generic exceptions and `models.ErrRankUnknown` +// returned by the committee when requesting the leader for the state's rank is propagated up the call stack. +// SafetyRules should *not* wrap unexpected exceptions into an expected NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceVote_CommitteeLeaderException() { + *s.committee = mocks.DynamicCommittee{} + for _, exception := range []error{ + errors.New("invalid-leader-identity"), + models.ErrRankUnknown, + } { + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return("", exception).Once() + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") + } +} + +// TestProduceVote_DifferentProposerFromLeader tests that no vote is created if the proposer is different from the leader for +// current rank. This is a byzantine behavior and should be handled by the compliance layer but nevertheless we want to +// have a sanity check for other code paths like voting on an own proposal created by the current leader. +func (s *SafetyRulesTestSuite) TestProduceVote_DifferentProposerFromLeader() { + s.proposal.State.ProposerID = helper.MakeIdentity() + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_NodeEjected tests that no vote is created if state proposer is ejected +func (s *SafetyRulesTestSuite) TestProduceVote_ProposerEjected() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity).Maybe() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.proposal.State.ProposerID).Return(nil, models.NewInvalidSignerErrorf("node-ejected")).Once() + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.proposerIdentity, nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.True(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_InvalidProposerIdentity tests that no vote is created if there was an exception retrieving proposer identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidProposerIdentity() { + *s.committee = mocks.DynamicCommittee{} + exception := errors.New("invalid-signer-identity") + s.committee.On("Self").Return(s.ourIdentity).Maybe() + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.proposerIdentity, nil).Once() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.proposal.State.ProposerID).Return(nil, exception).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_NodeNotAuthorizedToVote tests that no vote is created if the voter is not authorized to vote. +// Nodes have zero weight in the grace periods around the epochs where they are authorized to participate. +// We don't want zero-weight nodes to vote in the first place, to avoid unnecessary traffic. +// Note: this also covers ejected nodes. In both cases, the committee will return an `InvalidSignerError`. +func (s *SafetyRulesTestSuite) TestProduceVote_NodeEjected() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity) + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.proposerIdentity, nil).Once() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.proposal.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.ourIdentity).Return(nil, models.NewInvalidSignerErrorf("node-ejected")).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.True(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_InvalidVoterIdentity tests that no vote is created if there was an exception retrieving voter identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceVote_InvalidVoterIdentity() { + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity) + exception := errors.New("invalid-signer-identity") + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.proposerIdentity, nil).Once() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.proposal.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + s.committee.On("IdentityByState", s.proposal.State.Identifier, s.ourIdentity).Return(nil, exception).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_CreateVoteException tests that no vote is created if vote creation raised an exception +func (s *SafetyRulesTestSuite) TestProduceVote_CreateVoteException() { + exception := errors.New("create-vote-exception") + s.signer.On("CreateVote", s.proposal.State).Return(nil, exception).Once() + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_PersistStateException tests that no vote is created if persisting state failed +func (s *SafetyRulesTestSuite) TestProduceVote_PersistStateException() { + exception := errors.New("persister-exception") + s.persister.On("PutConsensusState", mock.Anything).Return(exception) + + vote := makeVote(s.proposal.State) + s.signer.On("CreateVote", s.proposal.State).Return(&vote, nil).Once() + votePtr, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Nil(s.T(), votePtr) + require.ErrorIs(s.T(), err, exception) +} + +// TestProduceVote_VotingOnInvalidProposals tests different scenarios where we try to vote on unsafe states +// SafetyRules contain a variety of checks to confirm that QC and TC have the desired relationship to each other. +// In particular, we test: +// +// (i) A TC should be included in a proposal, if and only of the QC is not the prior rank. +// (ii) When the proposal includes a TC (i.e. the QC not being for the prior rank), the TC must be for the prior rank. +// (iii) The QC in the state must have a smaller rank than the state. +// (iv) If the state contains a TC, the TC cannot contain a newer QC than the state itself. +// +// Conditions (i) - (iv) are validity requirements for the state and all states that SafetyRules processes +// are supposed to be pre-validated. Hence, failing any of those conditions means we have an internal bug. +// Consequently, we expect SafetyRules to return exceptions but _not_ `NoVoteError`, because the latter +// indicates that the input state was valid, but we didn't want to vote. +func (s *SafetyRulesTestSuite) TestProduceVote_VotingOnInvalidProposals() { + + // a proposal which includes a QC for the previous round should not contain a TC + s.Run("proposal-includes-last-rank-qc-and-tc", func() { + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+1))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState](helper.MakeTC())))) + s.committee.On("IdentityByState", proposal.State.Identifier, proposal.State.ProposerID).Return(&helper.TestWeightedIdentity{ID: s.proposerIdentity}, nil).Maybe() + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("no-last-rank-tc", func() { + // create state where State.Rank != State.ParentQuorumCertificate.GetRank()+1 and PreviousRankTimeoutCertificate = nil + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+2)))))) + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("last-rank-tc-invalid-rank", func() { + // create state where State.Rank != State.ParentQuorumCertificate.GetRank()+1 and + // State.Rank != PreviousRankTimeoutCertificate.Rank+1 + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+2))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState]( + helper.MakeTC( + helper.WithTCRank(s.bootstrapState.Rank)))))) + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("proposal-includes-QC-for-higher-rank", func() { + // create state where State.Rank != State.ParentQuorumCertificate.GetRank()+1 and + // State.Rank == PreviousRankTimeoutCertificate.Rank+1 and State.ParentQuorumCertificate.GetRank() >= State.Rank + // in this case state is not safe to extend since proposal includes QC which is newer than the proposal itself. + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+2), + func(state *models.State[*helper.TestState]) { + state.ParentQuorumCertificate = helper.MakeQC(helper.WithQCRank(s.bootstrapState.Rank + 10)) + })), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState]( + helper.MakeTC( + helper.WithTCRank(s.bootstrapState.Rank+1)))))) + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + s.Run("last-rank-tc-invalid-highest-qc", func() { + // create state where State.Rank != State.ParentQuorumCertificate.GetRank()+1 and + // State.Rank == PreviousRankTimeoutCertificate.Rank+1 and State.ParentQuorumCertificate.GetRank() < PreviousRankTimeoutCertificate.NewestQC.Rank + // in this case state is not safe to extend since proposal is built on top of QC, which is lower + // than QC presented in PreviousRankTimeoutCertificate. + TONewestQC := helper.MakeQC(helper.WithQCRank(s.bootstrapState.Rank + 1)) + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+2))), + helper.WithPreviousRankTimeoutCertificate[*helper.TestState]( + helper.MakeTC( + helper.WithTCRank(s.bootstrapState.Rank+1), + helper.WithTCNewestQC(TONewestQC)))))) + vote, err := s.safety.ProduceVote(proposal, proposal.State.Rank) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + }) + + s.signer.AssertNotCalled(s.T(), "CreateVote") + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceVote_VoteEquivocation tests scenario when we try to vote twice in same rank. We require that replica +// follows next rules: +// - replica votes once per rank +// - replica votes in monotonicly increasing ranks +// +// Voting twice per round on equivocating proposals is considered a byzantine behavior. +// Expect a `models.NoVoteError` sentinel in such scenario. +func (s *SafetyRulesTestSuite) TestProduceVote_VoteEquivocation() { + expectedVote := makeVote(s.proposal.State) + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", mock.Anything).Return(nil).Once() + + vote, err := s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + require.Equal(s.T(), &expectedVote, vote) + + equivocatingProposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal( + helper.WithState( + helper.MakeState( + helper.WithParentState(s.bootstrapState), + helper.WithStateRank[*helper.TestState](s.bootstrapState.Rank+1), + helper.WithStateProposer[*helper.TestState](s.proposerIdentity)), + )))) + + // voting at same rank(even different proposal) should result in NoVoteError + vote, err = s.safety.ProduceVote(equivocatingProposal, s.proposal.State.Rank) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + + s.proposal.State.ProposerID = s.ourIdentity + + // proposing at the same rank should result in NoVoteError since we have already voted + vote, err = s.safety.SignOwnProposal(&s.proposal.Proposal) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) +} + +// TestProduceVote_AfterTimeout tests a scenario where we first timeout for rank and then try to produce a vote for +// same rank, this should result in error since producing a timeout means that we have given up on this rank +// and are in process of moving forward, no vote should be created. +func (s *SafetyRulesTestSuite) TestProduceVote_AfterTimeout() { + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + expectedTimeout := &models.TimeoutState[*helper.TestVote]{ + Rank: rank, + LatestQuorumCertificate: newestQC, + } + s.signer.On("CreateTimeout", rank, newestQC, nil).Return(expectedTimeout, nil).Once() + s.persister.On("PutConsensusState", mock.Anything).Return(nil).Once() + + // first timeout, then try to vote + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.NoError(s.T(), err) + require.NotNil(s.T(), timeout) + + // voting in same rank after producing timeout is not allowed + vote, err := s.safety.ProduceVote(s.proposal, rank) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + + s.signer.AssertExpectations(s.T()) + s.persister.AssertExpectations(s.T()) +} + +// TestProduceTimeout_ShouldTimeout tests that we can produce timeout in cases where +// last rank was successful or not. Also tests last timeout caching. +func (s *SafetyRulesTestSuite) TestProduceTimeout_ShouldTimeout() { + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + expectedTimeout := &models.TimeoutState[*helper.TestVote]{ + Rank: rank, + LatestQuorumCertificate: newestQC, + // don't care about actual data + Vote: helper.MakeVote[*helper.TestVote](), + } + + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.safetyData.FinalizedRank, + LatestAcknowledgedRank: rank, + LatestTimeout: expectedTimeout, + } + s.signer.On("CreateTimeout", rank, newestQC, nil).Return(expectedTimeout, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.NoError(s.T(), err) + require.Equal(s.T(), expectedTimeout, timeout) + + s.persister.AssertCalled(s.T(), "PutConsensusState", expectedSafetyData) + + s.persister.On("PutConsensusState", mock.MatchedBy(func(s *models.ConsensusState[*helper.TestVote]) bool { + return s.LatestTimeout.TimeoutTick == 1 + })).Return(nil).Once() + + otherTimeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.NoError(s.T(), err) + require.True(s.T(), timeout.Equals(otherTimeout)) + require.Equal(s.T(), timeout.TimeoutTick+1, otherTimeout.TimeoutTick) + + // to create new TO we need to provide a TC + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(rank), + helper.WithTCNewestQC(newestQC)) + + expectedTimeout = &models.TimeoutState[*helper.TestVote]{ + Rank: rank + 1, + LatestQuorumCertificate: newestQC, + PriorRankTimeoutCertificate: previousRankTimeoutCert, + } + s.signer.On("CreateTimeout", rank+1, newestQC, previousRankTimeoutCert).Return(expectedTimeout, nil).Once() + expectedSafetyData = &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.safetyData.FinalizedRank, + LatestAcknowledgedRank: rank + 1, + LatestTimeout: expectedTimeout, + } + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + + // creating new timeout should invalidate cache + otherTimeout, err = s.safety.ProduceTimeout(rank+1, newestQC, previousRankTimeoutCert) + require.NoError(s.T(), err) + require.NotNil(s.T(), otherTimeout) +} + +// TestProduceTimeout_NotSafeToTimeout tests that we don't produce a timeout when it's not safe +// We expect that the EventHandler to feed only request timeouts for the current rank, providing valid set of inputs. +// Hence, the cases tested here would be symptoms of an internal bugs, and therefore should not result in an NoVoteError. +func (s *SafetyRulesTestSuite) TestProduceTimeout_NotSafeToTimeout() { + + s.Run("newest-qc-nil", func() { + // newestQC cannot be nil + timeout, err := s.safety.ProduceTimeout(s.safetyData.FinalizedRank, nil, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + // if a QC for the previous rank is provided, a last rank TC is unnecessary and must not be provided + s.Run("includes-last-rank-qc-and-tc", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + + // tc not needed but included + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+1, newestQC, helper.MakeTC()) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-rank-tc-nil", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + + // tc needed but not included + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+2, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-rank-tc-for-wrong-rank", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + // previousRankTimeoutCert should be for newestQC.GetRank()+1 + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(newestQC.GetRank())) + + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+2, newestQC, previousRankTimeoutCert) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-rank-equal-to-highest-QC", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(s.safetyData.FinalizedRank - 1)) + + timeout, err := s.safety.ProduceTimeout(s.safetyData.FinalizedRank, newestQC, previousRankTimeoutCert) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-rank-below-highest-QC", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(newestQC.GetRank() - 2)) + + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()-1, newestQC, previousRankTimeoutCert) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("last-rank-tc-is-newer", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + // newest QC included in TC cannot be higher than the newest QC known to replica + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(newestQC.GetRank()+1), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCRank(newestQC.GetRank()+1)))) + + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+2, newestQC, previousRankTimeoutCert) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("highest-qc-below-locked-round", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank - 1)) + + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+1, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + s.Run("cur-rank-below-highest-acknowledged-rank", func() { + newestQC := helper.MakeQC(helper.WithQCRank(s.safetyData.FinalizedRank)) + // modify highest acknowledged rank in a way that it's definitely bigger than the newest QC rank + s.safetyData.LatestAcknowledgedRank = newestQC.GetRank() + 10 + + timeout, err := s.safety.ProduceTimeout(newestQC.GetRank()+1, newestQC, nil) + require.Error(s.T(), err) + require.Nil(s.T(), timeout) + }) + + s.signer.AssertNotCalled(s.T(), "CreateTimeout") + s.signer.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceTimeout_CreateTimeoutException tests that no timeout is created if timeout creation raised an exception +func (s *SafetyRulesTestSuite) TestProduceTimeout_CreateTimeoutException() { + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + + exception := errors.New("create-timeout-exception") + s.signer.On("CreateTimeout", rank, newestQC, nil).Return(nil, exception).Once() + vote, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.Nil(s.T(), vote) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoVoteError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceTimeout_PersistStateException tests that no timeout is created if persisting state failed +func (s *SafetyRulesTestSuite) TestProduceTimeout_PersistStateException() { + exception := errors.New("persister-exception") + s.persister.On("PutConsensusState", mock.Anything).Return(exception) + + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + expectedTimeout := &models.TimeoutState[*helper.TestVote]{ + Rank: rank, + LatestQuorumCertificate: newestQC, + } + + s.signer.On("CreateTimeout", rank, newestQC, nil).Return(expectedTimeout, nil).Once() + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.Nil(s.T(), timeout) + require.ErrorIs(s.T(), err, exception) +} + +// TestProduceTimeout_AfterVote tests a case where we first produce a vote and then try to timeout +// for same rank. This behavior is expected and should result in valid timeout without any errors. +func (s *SafetyRulesTestSuite) TestProduceTimeout_AfterVote() { + expectedVote := makeVote(s.proposal.State) + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", mock.Anything).Return(nil).Times(2) + + rank := s.proposal.State.Rank + + // first produce vote, then try to timeout + vote, err := s.safety.ProduceVote(s.proposal, rank) + require.NoError(s.T(), err) + require.NotNil(s.T(), vote) + + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + + expectedTimeout := &models.TimeoutState[*helper.TestVote]{ + Rank: rank, + LatestQuorumCertificate: newestQC, + } + + s.signer.On("CreateTimeout", rank, newestQC, nil).Return(expectedTimeout, nil).Once() + + // timing out for same rank should be possible + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.NoError(s.T(), err) + require.NotNil(s.T(), timeout) + + s.persister.AssertExpectations(s.T()) + s.signer.AssertExpectations(s.T()) +} + +// TestProduceTimeout_InvalidProposerIdentity tests that no timeout is created if there was an exception retrieving proposer identity +// We are specifically testing that unexpected errors are handled correctly, i.e. +// that SafetyRules does not erroneously wrap unexpected exceptions into the expected models.NoTimeoutError. +func (s *SafetyRulesTestSuite) TestProduceTimeout_InvalidProposerIdentity() { + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + *s.committee = mocks.DynamicCommittee{} + exception := errors.New("invalid-signer-identity") + s.committee.On("IdentityByRank", rank, s.ourIdentity).Return(nil, exception).Once() + s.committee.On("Self").Return(s.ourIdentity) + + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.Nil(s.T(), timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsNoTimeoutError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestProduceTimeout_NodeEjected tests that no timeout is created if the replica is not authorized to create timeout. +// Nodes have zero weight in the grace periods around the epochs where they are authorized to participate. +// We don't want zero-weight nodes to participate in the first place, to avoid unnecessary traffic. +// Note: this also covers ejected nodes. In both cases, the committee will return an `InvalidSignerError`. +func (s *SafetyRulesTestSuite) TestProduceTimeout_NodeEjected() { + rank := s.proposal.State.Rank + newestQC := helper.MakeQC(helper.WithQCRank(rank - 1)) + *s.committee = mocks.DynamicCommittee{} + s.committee.On("Self").Return(s.ourIdentity) + s.committee.On("IdentityByRank", rank, s.ourIdentity).Return(nil, models.NewInvalidSignerErrorf("")).Maybe() + + timeout, err := s.safety.ProduceTimeout(rank, newestQC, nil) + require.Nil(s.T(), timeout) + require.True(s.T(), models.IsNoTimeoutError(err)) + s.persister.AssertNotCalled(s.T(), "PutConsensusState") +} + +// TestSignOwnProposal tests a happy path scenario where leader can sign their own proposal. +func (s *SafetyRulesTestSuite) TestSignOwnProposal() { + s.proposal.State.ProposerID = s.ourIdentity + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.proposal.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: s.proposal.State.Rank, + } + expectedVote := makeVote(s.proposal.State) + s.committee.On("LeaderForRank").Unset() + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.ourIdentity, nil).Once() + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + vote, err := s.safety.SignOwnProposal(&s.proposal.Proposal) + require.NoError(s.T(), err) + require.Equal(s.T(), vote, &expectedVote) +} + +// TestSignOwnProposal_ProposalNotSelf tests that we cannot sign a proposal that is not ours. We +// verify that SafetyRules returns an exception and not the benign sentinel error NoVoteError. +func (s *SafetyRulesTestSuite) TestSignOwnProposal_ProposalNotSelf() { + vote, err := s.safety.SignOwnProposal(&s.proposal.Proposal) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) +} + +// TestSignOwnProposal_SelfInvalidLeader tests that we cannot sign a proposal if we are not the leader for the rank. +// We verify that SafetyRules returns and exception and does not the benign sentinel error NoVoteError. +func (s *SafetyRulesTestSuite) TestSignOwnProposal_SelfInvalidLeader() { + s.proposal.State.ProposerID = s.ourIdentity + otherID := helper.MakeIdentity() + require.NotEqual(s.T(), otherID, s.ourIdentity) + s.committee.On("LeaderForRank").Unset() + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(otherID, nil).Once() + vote, err := s.safety.SignOwnProposal(&s.proposal.Proposal) + require.Error(s.T(), err) + require.False(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) +} + +// TestSignOwnProposal_ProposalEquivocation verifies that SafetyRules will refuse to sign multiple proposals for the same rank. +// We require that leader complies with the following next rules: +// - leader proposes once per rank +// - leader's proposals follow safety rules +// +// Signing repeatedly for one rank (either proposals or voting) can lead to equivocating (byzantine behavior). +// Expect a `models.NoVoteError` sentinel in such scenario. +func (s *SafetyRulesTestSuite) TestSignOwnProposal_ProposalEquivocation() { + s.proposal.State.ProposerID = s.ourIdentity + expectedSafetyData := &models.ConsensusState[*helper.TestVote]{ + FinalizedRank: s.proposal.State.ParentQuorumCertificate.GetRank(), + LatestAcknowledgedRank: s.proposal.State.Rank, + } + expectedVote := makeVote(s.proposal.State) + s.committee.On("LeaderForRank").Unset() + s.committee.On("LeaderForRank", s.proposal.State.Rank).Return(s.ourIdentity, nil).Once() + s.signer.On("CreateVote", s.proposal.State).Return(&expectedVote, nil).Once() + s.persister.On("PutConsensusState", expectedSafetyData).Return(nil).Once() + + vote, err := s.safety.SignOwnProposal(&s.proposal.Proposal) + require.NoError(s.T(), err) + require.Equal(s.T(), &expectedVote, vote) + + // signing same proposal again should return an error since we have already created a proposal for this rank + vote, err = s.safety.SignOwnProposal(&s.proposal.Proposal) + require.Error(s.T(), err) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) + + // voting for same rank should also return an error since we have already proposed + vote, err = s.safety.ProduceVote(s.proposal, s.proposal.State.Rank) + require.Error(s.T(), err) + require.True(s.T(), models.IsNoVoteError(err)) + require.Nil(s.T(), vote) +} + +func makeVote(state *models.State[*helper.TestState]) *helper.TestVote { + return &helper.TestVote{ + StateID: state.Identifier, + Rank: state.Rank, + ID: helper.MakeIdentity(), + } +} diff --git a/consensus/signature/state_signer_decoder.go b/consensus/signature/state_signer_decoder.go index 4e45d60..fc8ddae 100644 --- a/consensus/signature/state_signer_decoder.go +++ b/consensus/signature/state_signer_decoder.go @@ -29,7 +29,7 @@ var _ consensus.StateSignerDecoder[*nilUnique] = (*StateSignerDecoder[*nilUnique // validity of parent state. Consequently, the returned IdentifierList contains // the consensus participants that signed the parent state. Expected Error // returns during normal operations: -// - signature.InvalidSignerIndicesError if signer indices included in the +// - consensus.InvalidSignerIndicesError if signer indices included in the // state do not encode a valid subset of the consensus committee // - state.ErrUnknownSnapshotReference if the input state is not a known // incorporated state. diff --git a/consensus/signature/weighted_signature_aggregator.go b/consensus/signature/weighted_signature_aggregator.go index 1c3d54e..977ccc1 100644 --- a/consensus/signature/weighted_signature_aggregator.go +++ b/consensus/signature/weighted_signature_aggregator.go @@ -17,11 +17,11 @@ type signerInfo struct { } // WeightedSignatureAggregator implements consensus.WeightedSignatureAggregator. -// It is a wrapper around signature.SignatureAggregatorSameMessage, which +// It is a wrapper around consensus.SignatureAggregatorSameMessage, which // implements a mapping from node IDs (as used by HotStuff) to index-based // addressing of authorized signers (as used by SignatureAggregatorSameMessage). // -// Similarly to module/signature.SignatureAggregatorSameMessage, this module +// Similarly to module/consensus.SignatureAggregatorSameMessage, this module // assumes proofs of possession (PoP) of all identity public keys are valid. type WeightedSignatureAggregator struct { aggregator consensus.SignatureAggregator @@ -166,7 +166,7 @@ func (w *WeightedSignatureAggregator) TotalWeight() uint64 { return w.totalWeight } -// Aggregate aggregates the signatures and returns the aggregated signature. +// Aggregate aggregates the signatures and returns the aggregated consensus. // The function performs a final verification and errors if the aggregated // signature is invalid. This is required for the function safety since // `TrustedAdd` allows adding invalid signatures. The function errors with: diff --git a/consensus/state_machine.go b/consensus/state_machine.go index f0c83e7..83a4af4 100644 --- a/consensus/state_machine.go +++ b/consensus/state_machine.go @@ -1,1364 +1,1640 @@ package consensus -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/pkg/errors" -) - -// State represents a consensus engine state -type State string - -const ( - // StateStopped - Initial state, engine is not running - StateStopped State = "stopped" - // StateStarting - Engine is initializing - StateStarting State = "starting" - // StateLoading - Loading data and syncing with network - StateLoading State = "loading" - // StateCollecting - Collecting data for consensus round, prepares proposal - StateCollecting State = "collecting" - // StateLivenessCheck - Announces and awaits prover liveness - StateLivenessCheck State = "liveness_check" - // StateProving - Generating proof (prover only) - StateProving State = "proving" - // StatePublishing - Publishing relevant state - StatePublishing State = "publishing" - // StateVoting - Voting on proposals - StateVoting State = "voting" - // StateFinalizing - Finalizing consensus round - StateFinalizing State = "finalizing" - // StateVerifying - Verifying and publishing results - StateVerifying State = "verifying" - // StateStopping - Engine is shutting down - StateStopping State = "stopping" -) - -// Event represents an event that can trigger state transitions -type Event string - -const ( - EventStart Event = "start" - EventStop Event = "stop" - EventSyncTimeout Event = "sync_timeout" - EventInduceSync Event = "induce_sync" - EventSyncComplete Event = "sync_complete" - EventInitComplete Event = "init_complete" - EventCollectionDone Event = "collection_done" - EventLivenessCheckReceived Event = "liveness_check_received" - EventLivenessTimeout Event = "liveness_timeout" - EventProverSignal Event = "prover_signal" - EventProofComplete Event = "proof_complete" - EventPublishComplete Event = "publish_complete" - EventPublishTimeout Event = "publish_timeout" - EventProposalReceived Event = "proposal_received" - EventVoteReceived Event = "vote_received" - EventQuorumReached Event = "quorum_reached" - EventVotingTimeout Event = "voting_timeout" - EventAggregationDone Event = "aggregation_done" - EventAggregationTimeout Event = "aggregation_timeout" - EventConfirmationReceived Event = "confirmation_received" - EventVerificationDone Event = "verification_done" - EventVerificationTimeout Event = "verification_timeout" - EventCleanupComplete Event = "cleanup_complete" -) - -type Identity = string - -// Unique defines important attributes for distinguishing relative basis of -// items. -type Unique interface { - // Provides the relevant identity of the given Unique. - Identity() Identity - // Clone should provide a shallow clone of the Unique. - Clone() Unique - // Rank indicates the ordinal basis of comparison, e.g. a frame number, a - // height. - Rank() uint64 -} - -// TransitionGuard is a function that determines if a transition should occur -type TransitionGuard[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] func(sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT]) bool - -// Transition defines a state transition -type Transition[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] struct { - From State - Event Event - To State - Guard TransitionGuard[StateT, VoteT, PeerIDT, CollectedT] -} - -// TransitionListener is notified of state transitions -type TransitionListener[StateT Unique] interface { - OnTransition( - from State, - to State, - event Event, - ) -} - -type eventWrapper struct { - event Event - response chan error -} - -// SyncProvider handles synchronization management -type SyncProvider[StateT Unique] interface { - // Performs synchronization to set internal state. Note that it is assumed - // that errors are transient and synchronization should be reattempted on - // failure. If some other process for synchronization is used and this should - // be bypassed, send nil on the error channel. Provided context may be - // canceled, should be used to halt long-running sync operations. - Synchronize( - existing *StateT, - ctx context.Context, - ) (<-chan *StateT, <-chan error) -} - -// VotingProvider handles voting logic by deferring decisions, collection, and -// state finalization to an outside implementation. -type VotingProvider[StateT Unique, VoteT Unique, PeerIDT Unique] interface { - // Sends a proposal for voting. - SendProposal(proposal *StateT, ctx context.Context) error - // DecideAndSendVote makes a decision, mapped by leader, and should handle any - // side effects (like publishing vote messages). - DecideAndSendVote( - proposals map[Identity]*StateT, - ctx context.Context, - ) (PeerIDT, *VoteT, error) - // Re-publishes a vote message, used to help lagging peers catch up. - SendVote(vote *VoteT, ctx context.Context) (PeerIDT, error) - // IsQuorum returns a response indicating whether or not quorum has been - // reached. - IsQuorum( - proposalVotes map[Identity]*VoteT, - ctx context.Context, - ) (bool, error) - // FinalizeVotes performs any folding of proposed state required from VoteT - // onto StateT, proposed states and votes matched by PeerIDT, returns - // finalized state, chosen proposer PeerIDT. - FinalizeVotes( - proposals map[Identity]*StateT, - proposalVotes map[Identity]*VoteT, - ctx context.Context, - ) (*StateT, PeerIDT, error) - // SendConfirmation sends confirmation of the finalized state. - SendConfirmation(finalized *StateT, ctx context.Context) error -} - -// LeaderProvider handles leader selection. State is provided, if relevant to -// the upstream consensus engine. -type LeaderProvider[ - StateT Unique, - PeerIDT Unique, - CollectedT Unique, -] interface { - // GetNextLeaders returns a list of node indices, in priority order. Note that - // it is assumed that if no error is returned, GetNextLeaders should produce - // a non-empty list. If a list of size smaller than minimumProvers is - // provided, the liveness check will loop until the list is greater than that. - GetNextLeaders(prior *StateT, ctx context.Context) ([]PeerIDT, error) - // ProveNextState prepares a non-finalized new state from the prior, to be - // proposed and voted upon. Provided context may be canceled, should be used - // to halt long-running prover operations. - ProveNextState( - prior *StateT, - collected CollectedT, - ctx context.Context, - ) (*StateT, error) -} - -// LivenessProvider handles liveness announcements ahead of proving, to -// pre-emptively choose the next prover. In expected leader scenarios, this -// enables a peer to determine if an honest next prover is offline, so that it -// can publish the next state without waiting. -type LivenessProvider[ - StateT Unique, - PeerIDT Unique, - CollectedT Unique, -] interface { - // Collect returns the collected mutation operations ahead of liveness - // announcements. - Collect(ctx context.Context) (CollectedT, error) - // SendLiveness announces liveness ahead of the next prover deterimination and - // subsequent proving. Provides prior state and collected mutation operations - // if relevant. - SendLiveness(prior *StateT, collected CollectedT, ctx context.Context) error -} - -// TraceLogger defines a simple tracing interface -type TraceLogger interface { - Trace(message string) - Error(message string, err error) -} - -type nilTracer struct{} - -func (nilTracer) Trace(message string) {} -func (nilTracer) Error(message string, err error) {} - -// StateMachine manages consensus engine state transitions with generic state -// tracking. T represents the raw state bearing type, the implementation details -// are left to callers, who may augment their transitions to utilize the data -// if needed. If no method of fork choice is utilized external to this machine, -// this state machine provides BFT consensus (e.g. < 1/3 byzantine behaviors) -// provided assumptions outlined in interface types are fulfilled. The state -// transition patterns strictly assume a round-based state transition using -// cryptographic proofs. -// -// This implementation requires implementations of specific patterns: -// - A need to synchronize state from peers (SyncProvider) -// - A need to record voting from the upstream consumer to decide on consensus -// changes during the voting period (VotingProvider) -// - A need to decide on the next leader and prove (LeaderProvider) -// - A need to announce liveness ahead of long-running proof operations -// (LivenessProvider) -type StateMachine[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] struct { - mu sync.RWMutex - transitions map[State]map[Event]*Transition[ - StateT, VoteT, PeerIDT, CollectedT, - ] - stateConfigs map[State]*StateConfig[ - StateT, VoteT, PeerIDT, CollectedT, - ] - eventChan chan eventWrapper - ctx context.Context - cancel context.CancelFunc - timeoutTimer *time.Timer - behaviorCancel context.CancelFunc - - // Internal state - machineState State - activeState *StateT - nextState *StateT - collected *CollectedT - id PeerIDT - nextProvers []PeerIDT - liveness map[uint64]map[Identity]CollectedT - votes map[uint64]map[Identity]*VoteT - proposals map[uint64]map[Identity]*StateT - confirmations map[uint64]map[Identity]*StateT - chosenProposer *PeerIDT - stateStartTime time.Time - transitionCount uint64 - listeners []TransitionListener[StateT] - shouldEmitReceiveEventsOnSends bool - minimumProvers func() uint64 - - // Dependencies - syncProvider SyncProvider[StateT] - votingProvider VotingProvider[StateT, VoteT, PeerIDT] - leaderProvider LeaderProvider[StateT, PeerIDT, CollectedT] - livenessProvider LivenessProvider[StateT, PeerIDT, CollectedT] - traceLogger TraceLogger -} - -// StateConfig defines configuration for a state with generic behaviors -type StateConfig[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] struct { - // Callbacks for state entry/exit - OnEnter StateCallback[StateT, VoteT, PeerIDT, CollectedT] - OnExit StateCallback[StateT, VoteT, PeerIDT, CollectedT] - - // State behavior - runs continuously while in state - Behavior StateBehavior[StateT, VoteT, PeerIDT, CollectedT] - - // Timeout configuration - Timeout time.Duration - OnTimeout Event -} - -// StateCallback is called when entering or exiting a state -type StateCallback[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] func( - sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT], - data *StateT, - event Event, -) - -// StateBehavior defines the behavior while in a state -type StateBehavior[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -] func( - sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT], - data *StateT, - ctx context.Context, -) - -// NewStateMachine creates a new generic state machine for consensus. -// `initialState` should be provided if available, this does not set the -// position of the state machine however, consumers will need to manually force -// a state machine's internal state if desired. Assumes some variety of pubsub- -// based semantics are used in send/receive based operations, if the pubsub -// implementation chosen does not receive messages published by itself, set -// shouldEmitReceiveEventsOnSends to true. -func NewStateMachine[ - StateT Unique, - VoteT Unique, - PeerIDT Unique, - CollectedT Unique, -]( - id PeerIDT, - initialState *StateT, - shouldEmitReceiveEventsOnSends bool, - minimumProvers func() uint64, - syncProvider SyncProvider[StateT], - votingProvider VotingProvider[StateT, VoteT, PeerIDT], - leaderProvider LeaderProvider[StateT, PeerIDT, CollectedT], - livenessProvider LivenessProvider[StateT, PeerIDT, CollectedT], - traceLogger TraceLogger, -) *StateMachine[StateT, VoteT, PeerIDT, CollectedT] { - ctx, cancel := context.WithCancel(context.Background()) - if traceLogger == nil { - traceLogger = nilTracer{} - } - sm := &StateMachine[StateT, VoteT, PeerIDT, CollectedT]{ - machineState: StateStopped, - transitions: make( - map[State]map[Event]*Transition[StateT, VoteT, PeerIDT, CollectedT], - ), - stateConfigs: make( - map[State]*StateConfig[StateT, VoteT, PeerIDT, CollectedT], - ), - eventChan: make(chan eventWrapper, 100), - ctx: ctx, - cancel: cancel, - activeState: initialState, - id: id, - votes: make(map[uint64]map[Identity]*VoteT), - proposals: make(map[uint64]map[Identity]*StateT), - liveness: make(map[uint64]map[Identity]CollectedT), - confirmations: make(map[uint64]map[Identity]*StateT), - listeners: make([]TransitionListener[StateT], 0), - shouldEmitReceiveEventsOnSends: shouldEmitReceiveEventsOnSends, - minimumProvers: minimumProvers, - syncProvider: syncProvider, - votingProvider: votingProvider, - leaderProvider: leaderProvider, - livenessProvider: livenessProvider, - traceLogger: traceLogger, - } - - // Define state configurations - sm.defineStateConfigs() - - // Define transitions - sm.defineTransitions() - - // Start event processor - go sm.processEvents() - - return sm -} - -// defineStateConfigs sets up state configurations with behaviors -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) defineStateConfigs() { - sm.traceLogger.Trace("enter defineStateConfigs") - defer sm.traceLogger.Trace("exit defineStateConfigs") - // Starting state - just timeout to complete initialization - sm.stateConfigs[StateStarting] = &StateConfig[ - StateT, - VoteT, - PeerIDT, - CollectedT, - ]{ - Timeout: 1 * time.Second, - OnTimeout: EventInitComplete, - } - - type Config = StateConfig[ - StateT, - VoteT, - PeerIDT, - CollectedT, - ] - - type SMT = StateMachine[StateT, VoteT, PeerIDT, CollectedT] - - // Loading state - synchronize with network - sm.stateConfigs[StateLoading] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Loading behavior") - defer sm.traceLogger.Trace("exit Loading behavior") - if sm.syncProvider != nil { - newStateCh, errCh := sm.syncProvider.Synchronize(sm.activeState, ctx) - select { - case newState := <-newStateCh: - sm.mu.Lock() - sm.activeState = newState - sm.mu.Unlock() - nextLeaders, err := sm.leaderProvider.GetNextLeaders(newState, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - time.Sleep(10 * time.Second) - sm.SendEvent(EventSyncTimeout) - return - } - found := false - for _, leader := range nextLeaders { - if leader.Identity() == sm.id.Identity() { - found = true - break - } - } - if found { - sm.SendEvent(EventSyncComplete) - } else { - time.Sleep(10 * time.Second) - sm.SendEvent(EventSyncTimeout) - } - case <-errCh: - time.Sleep(10 * time.Second) - sm.SendEvent(EventSyncTimeout) - case <-ctx.Done(): - return - } - } - }, - Timeout: 10 * time.Hour, - OnTimeout: EventSyncTimeout, - } - - // Collecting state - wait for frame or timeout - sm.stateConfigs[StateCollecting] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Collecting behavior") - defer sm.traceLogger.Trace("exit Collecting behavior") - collected, err := sm.livenessProvider.Collect(ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - - sm.mu.Lock() - sm.nextProvers = []PeerIDT{} - sm.chosenProposer = nil - sm.collected = &collected - sm.mu.Unlock() - - nextProvers, err := sm.leaderProvider.GetNextLeaders(data, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - - sm.mu.Lock() - sm.nextProvers = nextProvers - sm.mu.Unlock() - - err = sm.livenessProvider.SendLiveness(data, collected, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - - sm.mu.Lock() - if sm.shouldEmitReceiveEventsOnSends { - if _, ok := sm.liveness[collected.Rank()]; !ok { - sm.liveness[collected.Rank()] = make(map[Identity]CollectedT) - } - sm.liveness[collected.Rank()][sm.id.Identity()] = *sm.collected - } - sm.mu.Unlock() - - if sm.shouldEmitReceiveEventsOnSends { - sm.SendEvent(EventLivenessCheckReceived) - } - - sm.SendEvent(EventCollectionDone) - }, - Timeout: 10 * time.Second, - OnTimeout: EventInduceSync, - } - - // Liveness check state - sm.stateConfigs[StateLivenessCheck] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Liveness behavior") - defer sm.traceLogger.Trace("exit Liveness behavior") - sm.mu.Lock() - nextProversLen := len(sm.nextProvers) - sm.mu.Unlock() - - // If we're not meeting the minimum prover count, we should loop. - if nextProversLen < int(sm.minimumProvers()) { - sm.traceLogger.Trace("insufficient provers, re-fetching leaders") - var err error - nextProvers, err := sm.leaderProvider.GetNextLeaders(data, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - sm.mu.Lock() - sm.nextProvers = nextProvers - sm.mu.Unlock() - } - - sm.mu.Lock() - collected := *sm.collected - sm.mu.Unlock() - - sm.mu.Lock() - livenessLen := len(sm.liveness[(*sm.activeState).Rank()+1]) - sm.mu.Unlock() - - // We have enough checks for consensus: - if livenessLen >= int(sm.minimumProvers()) { - sm.traceLogger.Trace( - "sufficient liveness checks, sending prover signal", - ) - sm.SendEvent(EventProverSignal) - return - } - - sm.traceLogger.Trace( - fmt.Sprintf( - "insufficient liveness checks: need %d, have %d", - sm.minimumProvers(), - livenessLen, - ), - ) - - select { - case <-time.After(1 * time.Second): - err := sm.livenessProvider.SendLiveness(data, collected, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - case <-ctx.Done(): - } - }, - Timeout: 2 * time.Second, - OnTimeout: EventLivenessTimeout, - } - - // Proving state - generate proof - sm.stateConfigs[StateProving] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Proving behavior") - defer sm.traceLogger.Trace("exit Proving behavior") - sm.mu.Lock() - collected := sm.collected - sm.collected = nil - sm.mu.Unlock() - - if collected == nil { - sm.SendEvent(EventInduceSync) - return - } - - proposal, err := sm.leaderProvider.ProveNextState( - data, - *collected, - ctx, - ) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - - sm.SendEvent(EventInduceSync) - return - } - - sm.mu.Lock() - sm.traceLogger.Trace( - fmt.Sprintf("adding proposal with rank %d", (*proposal).Rank()), - ) - if _, ok := sm.proposals[(*proposal).Rank()]; !ok { - sm.proposals[(*proposal).Rank()] = make(map[Identity]*StateT) - } - sm.proposals[(*proposal).Rank()][sm.id.Identity()] = proposal - sm.mu.Unlock() - - sm.SendEvent(EventProofComplete) - }, - Timeout: 120 * time.Second, - OnTimeout: EventPublishTimeout, - } - - // Publishing state - publish frame - sm.stateConfigs[StatePublishing] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Publishing behavior") - defer sm.traceLogger.Trace("exit Publishing behavior") - sm.mu.Lock() - if _, ok := sm.proposals[(*data).Rank()+1][sm.id.Identity()]; ok { - proposal := sm.proposals[(*data).Rank()+1][sm.id.Identity()] - sm.mu.Unlock() - - err := sm.votingProvider.SendProposal( - proposal, - ctx, - ) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - sm.SendEvent(EventPublishComplete) - } else { - sm.mu.Unlock() - } - }, - Timeout: 1 * time.Second, - OnTimeout: EventPublishTimeout, - } - - // Voting state - monitor for quorum - sm.stateConfigs[StateVoting] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Voting behavior") - defer sm.traceLogger.Trace("exit Voting behavior") - - sm.mu.Lock() - - if sm.chosenProposer == nil { - // We haven't voted yet - sm.traceLogger.Trace("proposer not yet chosen") - perfect := map[int]PeerIDT{} // all provers - live := map[int]PeerIDT{} // the provers who told us they're alive - for i, p := range sm.nextProvers { - perfect[i] = p - if _, ok := sm.liveness[(*sm.activeState).Rank()+1][p.Identity()]; ok { - live[i] = p - } - } - - if len(sm.proposals[(*sm.activeState).Rank()+1]) < int(sm.minimumProvers()) { - sm.traceLogger.Trace( - fmt.Sprintf( - "insufficient proposal count: %d, need %d", - len(sm.proposals[(*sm.activeState).Rank()+1]), - int(sm.minimumProvers()), - ), - ) - sm.mu.Unlock() - return - } - - if ctx == nil { - sm.traceLogger.Trace("context null") - sm.mu.Unlock() - return - } - - select { - case <-ctx.Done(): - sm.traceLogger.Trace("context canceled") - sm.mu.Unlock() - return - default: - sm.traceLogger.Trace("choosing proposal") - proposals := map[Identity]*StateT{} - for k, v := range sm.proposals[(*sm.activeState).Rank()+1] { - state := (*v).Clone().(StateT) - proposals[k] = &state - } - - sm.mu.Unlock() - selectedPeer, vote, err := sm.votingProvider.DecideAndSendVote( - proposals, - ctx, - ) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - break - } - sm.mu.Lock() - sm.chosenProposer = &selectedPeer - - if sm.shouldEmitReceiveEventsOnSends { - if _, ok := sm.votes[(*sm.activeState).Rank()+1]; !ok { - sm.votes[(*sm.activeState).Rank()+1] = make(map[Identity]*VoteT) - } - sm.votes[(*sm.activeState).Rank()+1][sm.id.Identity()] = vote - sm.mu.Unlock() - sm.SendEvent(EventVoteReceived) - return - } - sm.mu.Unlock() - } - } else { - sm.traceLogger.Trace("proposal chosen, checking for quorum") - proposalVotes := map[Identity]*VoteT{} - for p, vp := range sm.votes[(*sm.activeState).Rank()+1] { - vclone := (*vp).Clone().(VoteT) - proposalVotes[p] = &vclone - } - haveEnoughProposals := len(sm.proposals[(*sm.activeState).Rank()+1]) >= - int(sm.minimumProvers()) - sm.mu.Unlock() - isQuorum, err := sm.votingProvider.IsQuorum(proposalVotes, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - - if isQuorum && haveEnoughProposals { - sm.traceLogger.Trace("quorum reached") - sm.SendEvent(EventQuorumReached) - } else { - sm.traceLogger.Trace( - fmt.Sprintf( - "quorum not reached: proposals: %d, needed: %d", - len(sm.proposals[(*sm.activeState).Rank()+1]), - sm.minimumProvers(), - ), - ) - } - } - }, - Timeout: 1 * time.Second, - OnTimeout: EventVotingTimeout, - } - - // Finalizing state - sm.stateConfigs[StateFinalizing] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.mu.Lock() - proposals := map[Identity]*StateT{} - for k, v := range sm.proposals[(*sm.activeState).Rank()+1] { - state := (*v).Clone().(StateT) - proposals[k] = &state - } - proposalVotes := map[Identity]*VoteT{} - for p, vp := range sm.votes[(*sm.activeState).Rank()+1] { - vclone := (*vp).Clone().(VoteT) - proposalVotes[p] = &vclone - } - sm.mu.Unlock() - finalized, _, err := sm.votingProvider.FinalizeVotes( - proposals, - proposalVotes, - ctx, - ) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - next := (*finalized).Clone().(StateT) - sm.mu.Lock() - sm.nextState = &next - sm.mu.Unlock() - sm.SendEvent(EventAggregationDone) - }, - Timeout: 1 * time.Second, - OnTimeout: EventAggregationTimeout, - } - - // Verifying state - sm.stateConfigs[StateVerifying] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.traceLogger.Trace("enter Verifying behavior") - defer sm.traceLogger.Trace("exit Verifying behavior") - sm.mu.Lock() - if _, ok := sm.confirmations[(*sm.activeState).Rank()+1][sm.id.Identity()]; !ok && - sm.nextState != nil { - nextState := sm.nextState - sm.mu.Unlock() - err := sm.votingProvider.SendConfirmation(nextState, ctx) - if err != nil { - sm.traceLogger.Error( - fmt.Sprintf("error encountered in %s", sm.machineState), - err, - ) - sm.SendEvent(EventInduceSync) - return - } - sm.mu.Lock() - } - - progressed := false - if sm.nextState != nil { - sm.activeState = sm.nextState - progressed = true - } - if progressed { - sm.nextState = nil - sm.collected = nil - delete(sm.liveness, (*sm.activeState).Rank()) - delete(sm.proposals, (*sm.activeState).Rank()) - delete(sm.votes, (*sm.activeState).Rank()) - delete(sm.confirmations, (*sm.activeState).Rank()) - sm.mu.Unlock() - sm.SendEvent(EventVerificationDone) - } else { - sm.mu.Unlock() - } - }, - Timeout: 1 * time.Second, - OnTimeout: EventVerificationTimeout, - } - - // Stopping state - sm.stateConfigs[StateStopping] = &Config{ - Behavior: func(sm *SMT, data *StateT, ctx context.Context) { - sm.SendEvent(EventCleanupComplete) - }, - Timeout: 30 * time.Second, - OnTimeout: EventCleanupComplete, - } -} - -// defineTransitions sets up all possible state transitions -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) defineTransitions() { - sm.traceLogger.Trace("enter defineTransitions") - defer sm.traceLogger.Trace("exit defineTransitions") - - // Helper to add transition - addTransition := func( - from State, - event Event, - to State, - guard TransitionGuard[StateT, VoteT, PeerIDT, CollectedT], - ) { - if sm.transitions[from] == nil { - sm.transitions[from] = make(map[Event]*Transition[ - StateT, - VoteT, - PeerIDT, - CollectedT, - ]) - } - sm.transitions[from][event] = &Transition[ - StateT, - VoteT, - PeerIDT, - CollectedT, - ]{ - From: from, - Event: event, - To: to, - Guard: guard, - } - } - - // Basic flow transitions - addTransition(StateStopped, EventStart, StateStarting, nil) - addTransition(StateStarting, EventInitComplete, StateLoading, nil) - addTransition(StateLoading, EventSyncTimeout, StateLoading, nil) - addTransition(StateLoading, EventSyncComplete, StateCollecting, nil) - addTransition(StateCollecting, EventCollectionDone, StateLivenessCheck, nil) - addTransition(StateLivenessCheck, EventProverSignal, StateProving, nil) - - // Loop indefinitely if nobody can be found - addTransition( - StateLivenessCheck, - EventLivenessTimeout, - StateLivenessCheck, - nil, - ) - // // Loop until we get enough of these - // addTransition( - // StateLivenessCheck, - // EventLivenessCheckReceived, - // StateLivenessCheck, - // nil, - // ) - - // Prover flow - addTransition(StateProving, EventProofComplete, StatePublishing, nil) - addTransition(StateProving, EventPublishTimeout, StateVoting, nil) - addTransition(StatePublishing, EventPublishComplete, StateVoting, nil) - addTransition(StatePublishing, EventPublishTimeout, StateVoting, nil) - - // Common voting flow - addTransition(StateVoting, EventProposalReceived, StateVoting, nil) - // addTransition(StateVoting, EventVoteReceived, StateVoting, nil) - addTransition(StateVoting, EventQuorumReached, StateFinalizing, nil) - addTransition(StateVoting, EventVotingTimeout, StateVoting, nil) - addTransition(StateFinalizing, EventAggregationDone, StateVerifying, nil) - addTransition(StateFinalizing, EventAggregationTimeout, StateFinalizing, nil) - addTransition(StateVerifying, EventVerificationDone, StateCollecting, nil) - addTransition(StateVerifying, EventVerificationTimeout, StateVerifying, nil) - - // Stop or induce Sync transitions from any state - for _, state := range []State{ - StateStarting, - StateLoading, - StateCollecting, - StateLivenessCheck, - StateProving, - StatePublishing, - StateVoting, - StateFinalizing, - StateVerifying, - } { - addTransition(state, EventStop, StateStopping, nil) - addTransition(state, EventInduceSync, StateLoading, nil) - } - - addTransition(StateStopping, EventCleanupComplete, StateStopped, nil) -} - -// Start begins the state machine -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) Start() error { - sm.traceLogger.Trace("enter start") - defer sm.traceLogger.Trace("exit start") - sm.SendEvent(EventStart) - return nil -} - -// Stop halts the state machine -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) Stop() error { - sm.traceLogger.Trace("enter stop") - defer sm.traceLogger.Trace("exit stop") -drain: - for { - select { - case <-sm.eventChan: - default: - break drain - } - } - sm.SendEvent(EventStop) - return nil -} - -// SendEvent sends an event to the state machine -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) SendEvent(event Event) { - sm.traceLogger.Trace(fmt.Sprintf("enter sendEvent: %s", event)) - defer sm.traceLogger.Trace(fmt.Sprintf("exit sendEvent: %s", event)) - response := make(chan error, 1) - go func() { - select { - case sm.eventChan <- eventWrapper{event: event, response: response}: - <-response - case <-sm.ctx.Done(): - return - } - }() -} - -// processEvents handles events and transitions -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) processEvents() { - defer func() { - if r := recover(); r != nil { - sm.traceLogger.Error( - "fatal error encountered", - errors.New(fmt.Sprintf("%+v", r)), - ) - sm.Close() - } - }() - - sm.traceLogger.Trace("enter processEvents") - defer sm.traceLogger.Trace("exit processEvents") - for { - select { - case <-sm.ctx.Done(): - return - case wrapper := <-sm.eventChan: - err := sm.handleEvent(wrapper.event) - wrapper.response <- err - } - } -} - -// handleEvent processes a single event -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) handleEvent(event Event) error { - sm.traceLogger.Trace(fmt.Sprintf("enter handleEvent: %s", event)) - defer sm.traceLogger.Trace(fmt.Sprintf("exit handleEvent: %s", event)) - sm.mu.Lock() - - currentState := sm.machineState - transitions, exists := sm.transitions[currentState] - if !exists { - sm.mu.Unlock() - - return errors.Wrap( - fmt.Errorf("no transitions defined for state %s", currentState), - "handle event", - ) - } - - transition, exists := transitions[event] - if !exists { - sm.mu.Unlock() - - return errors.Wrap( - fmt.Errorf( - "no transition for event %s in state %s", - event, - currentState, - ), - "handle event", - ) - } - - // Check guard condition with the actual state - if transition.Guard != nil && !transition.Guard(sm) { - sm.mu.Unlock() - - return errors.Wrap( - fmt.Errorf( - "transition guard failed for %s -> %s on %s", - currentState, - transition.To, - event, - ), - "handle event", - ) - } - - sm.mu.Unlock() - - // Execute transition - sm.executeTransition(currentState, transition.To, event) - return nil -} - -// executeTransition performs the state transition -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) executeTransition( - from State, - to State, - event Event, -) { - sm.traceLogger.Trace( - fmt.Sprintf("enter executeTransition: %s -> %s [%s]", from, to, event), - ) - defer sm.traceLogger.Trace( - fmt.Sprintf("exit executeTransition: %s -> %s [%s]", from, to, event), - ) - sm.mu.Lock() - - // Cancel any existing timeout and behavior - if sm.timeoutTimer != nil { - sm.timeoutTimer.Stop() - sm.timeoutTimer = nil - } - - // Cancel existing behavior if any - if sm.behaviorCancel != nil { - sm.behaviorCancel() - sm.behaviorCancel = nil - } - - // Call exit callback for current state - if config, exists := sm.stateConfigs[from]; exists && config.OnExit != nil { - sm.mu.Unlock() - config.OnExit(sm, sm.activeState, event) - sm.mu.Lock() - } - - // Update state - sm.machineState = to - sm.stateStartTime = time.Now() - sm.transitionCount++ - - // Notify listeners - for _, listener := range sm.listeners { - listener.OnTransition(from, to, event) - } - - // Call enter callback for new state - if config, exists := sm.stateConfigs[to]; exists { - if config.OnEnter != nil { - sm.mu.Unlock() - config.OnEnter(sm, sm.activeState, event) - sm.mu.Lock() - } - - // Start state behavior if defined - if config.Behavior != nil { - behaviorCtx, cancel := context.WithCancel(sm.ctx) - sm.behaviorCancel = cancel - sm.mu.Unlock() - config.Behavior(sm, sm.activeState, behaviorCtx) - sm.mu.Lock() - } - - // Set up timeout for new state - if config.Timeout > 0 && config.OnTimeout != "" { - sm.timeoutTimer = time.AfterFunc(config.Timeout, func() { - sm.SendEvent(config.OnTimeout) - }) - } - } - sm.mu.Unlock() -} - -// GetState returns the current state -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) GetState() State { - sm.traceLogger.Trace("enter getstate") - defer sm.traceLogger.Trace("exit getstate") - sm.mu.Lock() - defer sm.mu.Unlock() - return sm.machineState -} - -// Additional methods for compatibility -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) GetStateTime() time.Duration { - sm.traceLogger.Trace("enter getstatetime") - defer sm.traceLogger.Trace("exit getstatetime") - return time.Since(sm.stateStartTime) -} - -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) GetTransitionCount() uint64 { - sm.traceLogger.Trace("enter transitioncount") - defer sm.traceLogger.Trace("exit transitioncount") - return sm.transitionCount -} - -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) AddListener(listener TransitionListener[StateT]) { - sm.traceLogger.Trace("enter addlistener") - defer sm.traceLogger.Trace("exit addlistener") - sm.mu.Lock() - defer sm.mu.Unlock() - sm.listeners = append(sm.listeners, listener) -} - -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) Close() { - sm.traceLogger.Trace("enter close") - defer sm.traceLogger.Trace("exit close") - sm.mu.Lock() - defer sm.mu.Unlock() - sm.cancel() - if sm.timeoutTimer != nil { - sm.timeoutTimer.Stop() - } - if sm.behaviorCancel != nil { - sm.behaviorCancel() - } - sm.machineState = StateStopped -} - -// ReceiveLivenessCheck receives a liveness announcement and captures -// collected mutation operations reported by the peer if relevant. -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) ReceiveLivenessCheck(peer PeerIDT, collected CollectedT) error { - sm.traceLogger.Trace( - fmt.Sprintf( - "enter receivelivenesscheck, peer: %s, rank: %d", - peer.Identity(), - collected.Rank(), - ), - ) - defer sm.traceLogger.Trace("exit receivelivenesscheck") - sm.mu.Lock() - if _, ok := sm.liveness[collected.Rank()]; !ok { - sm.liveness[collected.Rank()] = make(map[Identity]CollectedT) - } - if _, ok := sm.liveness[collected.Rank()][peer.Identity()]; !ok { - sm.liveness[collected.Rank()][peer.Identity()] = collected - } - sm.mu.Unlock() - - sm.SendEvent(EventLivenessCheckReceived) - return nil -} - -// ReceiveProposal receives a proposed new state. -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) ReceiveProposal(peer PeerIDT, proposal *StateT) error { - sm.traceLogger.Trace("enter receiveproposal") - defer sm.traceLogger.Trace("exit receiveproposal") - sm.mu.Lock() - if _, ok := sm.proposals[(*proposal).Rank()]; !ok { - sm.proposals[(*proposal).Rank()] = make(map[Identity]*StateT) - } - if _, ok := sm.proposals[(*proposal).Rank()][peer.Identity()]; !ok { - sm.proposals[(*proposal).Rank()][peer.Identity()] = proposal - } - sm.mu.Unlock() - - sm.SendEvent(EventProposalReceived) - return nil -} - -// ReceiveVote captures a vote. Presumes structural and protocol validity of a -// vote has already been evaluated. -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) ReceiveVote(proposer PeerIDT, voter PeerIDT, vote *VoteT) error { - sm.traceLogger.Trace("enter receivevote") - defer sm.traceLogger.Trace("exit receivevote") - sm.mu.Lock() - - if _, ok := sm.votes[(*vote).Rank()]; !ok { - sm.votes[(*vote).Rank()] = make(map[Identity]*VoteT) - } - if _, ok := sm.votes[(*vote).Rank()][voter.Identity()]; !ok { - sm.votes[(*vote).Rank()][voter.Identity()] = vote - } else if (*sm.votes[(*vote).Rank()][voter.Identity()]).Identity() != - (*vote).Identity() { - sm.mu.Unlock() - return errors.Wrap(errors.New("received conflicting vote"), "receive vote") - } - sm.mu.Unlock() - - sm.SendEvent(EventVoteReceived) - return nil -} - -// ReceiveConfirmation captures a confirmation. Presumes structural and protocol -// validity of the state has already been evaluated. -func (sm *StateMachine[ - StateT, - VoteT, - PeerIDT, - CollectedT, -]) ReceiveConfirmation( - peer PeerIDT, - confirmation *StateT, -) error { - sm.traceLogger.Trace("enter receiveconfirmation") - defer sm.traceLogger.Trace("exit receiveconfirmation") - sm.mu.Lock() - if _, ok := sm.confirmations[(*confirmation).Rank()]; !ok { - sm.confirmations[(*confirmation).Rank()] = make(map[Identity]*StateT) - } - if _, ok := sm.confirmations[(*confirmation).Rank()][peer.Identity()]; !ok { - sm.confirmations[(*confirmation).Rank()][peer.Identity()] = confirmation - } - sm.mu.Unlock() - - sm.SendEvent(EventConfirmationReceived) - return nil -} +// import ( +// "context" +// "fmt" +// "sync" +// "sync/atomic" +// "time" + +// "github.com/pkg/errors" +// "source.quilibrium.com/quilibrium/monorepo/consensus/models" +// ) + +// // // TransitionGuard is a function that determines if a transition should occur +// // type TransitionGuard[ +// // StateT Unique, +// // VoteT Unique, +// // PeerIDT Unique, +// // CollectedT Unique, +// // ] func(sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT]) bool + +// // // Transition defines a state transition +// // type Transition[ +// // StateT Unique, +// // VoteT Unique, +// // PeerIDT Unique, +// // CollectedT Unique, +// // ] struct { +// // From State +// // Event Event +// // To State +// // Guard TransitionGuard[StateT, VoteT, PeerIDT, CollectedT] +// // } + +// // // TransitionListener is notified of state transitions +// // type TransitionListener[StateT Unique] interface { +// // OnTransition( +// // from State, +// // to State, +// // event Event, +// // ) +// // } + +// // type eventWrapper struct { +// // event Event +// // response chan error +// // } + +// type proposal[StateT Unique] struct { +// proposal StateT +// time time.Time +// } + +// // StateMachine manages consensus engine state transitions with generic state +// // tracking. T represents the raw state bearing type, the implementation details +// // are left to callers, who may augment their transitions to utilize the data +// // if needed. If no method of fork choice is utilized external to this machine, +// // this state machine provides BFT consensus (e.g. < 1/3 byzantine behaviors) +// // provided assumptions outlined in interface types are fulfilled. The state +// // transition patterns strictly assume a round-based state transition using +// // cryptographic proofs. +// // +// // This implementation requires implementations of specific patterns: +// // - A need to synchronize state from peers (SyncProvider) +// // - A need to record voting from the upstream consumer to decide on consensus +// // changes during the voting period (VotingProvider) +// // - A need to decide on the next leader and prove (LeaderProvider) +// // - A need to announce liveness ahead of long-running proof operations +// // (LivenessProvider) +// type StateMachine[ +// StateT Unique, +// VoteT Unique, +// PeerIDT Unique, +// CollectedT Unique, +// ] struct { +// mu sync.RWMutex +// // transitions map[State]map[Event]*Transition[ +// // StateT, VoteT, PeerIDT, CollectedT, +// // ] +// // stateConfigs map[State]*StateConfig[ +// // StateT, VoteT, PeerIDT, CollectedT, +// // ] +// eventChan chan eventWrapper +// ctx context.Context +// cancel context.CancelFunc +// timeoutTimer *time.Timer +// behaviorCancel context.CancelFunc + +// // Internal state +// machineState State +// activeState *StateT +// nextState *StateT +// collected *CollectedT +// id PeerIDT +// nextProvers []PeerIDT +// // liveness map[uint64]map[Identity]CollectedT +// // votes map[uint64]map[Identity]*VoteT +// // proposals map[uint64]map[Identity]*StateT +// // confirmations map[uint64]map[Identity]*StateT +// chosenProposer *PeerIDT +// stateStartTime time.Time +// transitionCount uint64 +// // listeners []TransitionListener[StateT] +// shouldEmitReceiveEventsOnSends bool +// minimumProvers func() uint64 +// proposals chan proposal[StateT] +// latestTimeoutCertificate *atomic.Value +// latestQuorumCertificate *atomic.Value +// latestPartialTimeoutCertificate *atomic.Value +// timeoutCertificateCh chan models.TimeoutCertificate +// quorumCertificateCh chan models.QuorumCertificate +// partialTimeoutCertificateCh chan models.TimeoutCertificate +// startTime time.Time + +// // Dependencies +// syncProvider SyncProvider[StateT] +// votingProvider VotingProvider[StateT, VoteT, PeerIDT] +// leaderProvider LeaderProvider[StateT, PeerIDT, CollectedT] +// livenessProvider LivenessProvider[StateT, PeerIDT, CollectedT] +// pacemakerProvider PacemakerProvider +// traceLogger TraceLogger +// } + +// // StateConfig defines configuration for a state with generic behaviors +// // type StateConfig[ +// // StateT Unique, +// // VoteT Unique, +// // PeerIDT Unique, +// // CollectedT Unique, +// // ] struct { +// // // Callbacks for state entry/exit +// // OnEnter StateCallback[StateT, VoteT, PeerIDT, CollectedT] +// // OnExit StateCallback[StateT, VoteT, PeerIDT, CollectedT] + +// // // State behavior - runs continuously while in state +// // Behavior StateBehavior[StateT, VoteT, PeerIDT, CollectedT] + +// // // Timeout configuration +// // Timeout time.Duration +// // OnTimeout Event +// // } + +// // // StateCallback is called when entering or exiting a state +// // type StateCallback[ +// // StateT Unique, +// // VoteT Unique, +// // PeerIDT Unique, +// // CollectedT Unique, +// // ] func( +// // sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT], +// // data *StateT, +// // event Event, +// // ) + +// // // StateBehavior defines the behavior while in a state +// // type StateBehavior[ +// // StateT Unique, +// // VoteT Unique, +// // PeerIDT Unique, +// // CollectedT Unique, +// // ] func( +// // sm *StateMachine[StateT, VoteT, PeerIDT, CollectedT], +// // data *StateT, +// // ctx context.Context, +// // ) + +// // NewStateMachine creates a new generic state machine for consensus. +// // `initialState` should be provided if available, this does not set the +// // position of the state machine however, consumers will need to manually force +// // a state machine's internal state if desired. Assumes some variety of pubsub- +// // based semantics are used in send/receive based operations, if the pubsub +// // implementation chosen does not receive messages published by itself, set +// // shouldEmitReceiveEventsOnSends to true. +// func NewStateMachine[ +// StateT Unique, +// VoteT Unique, +// PeerIDT Unique, +// CollectedT Unique, +// ]( +// id PeerIDT, +// shouldEmitReceiveEventsOnSends bool, +// minimumProvers func() uint64, +// syncProvider SyncProvider[StateT], +// votingProvider VotingProvider[StateT, VoteT, PeerIDT], +// leaderProvider LeaderProvider[StateT, PeerIDT, CollectedT], +// livenessProvider LivenessProvider[StateT, PeerIDT, CollectedT], +// pacemakerProvider PacemakerProvider, +// traceLogger TraceLogger, +// ) *StateMachine[StateT, VoteT, PeerIDT, CollectedT] { +// ctx, cancel := context.WithCancel(context.Background()) +// if traceLogger == nil { +// traceLogger = nilTracer{} +// } +// sm := &StateMachine[StateT, VoteT, PeerIDT, CollectedT]{ +// // transitions: make( +// // map[State]map[Event]*Transition[StateT, VoteT, PeerIDT, CollectedT], +// // ), +// // stateConfigs: make( +// // map[State]*StateConfig[StateT, VoteT, PeerIDT, CollectedT], +// // ), +// ctx: ctx, +// cancel: cancel, +// id: id, +// // votes: make(map[uint64]map[Identity]*VoteT), +// // proposals: make(map[uint64]map[Identity]*StateT), +// // liveness: make(map[uint64]map[Identity]CollectedT), +// // confirmations: make(map[uint64]map[Identity]*StateT), +// // listeners: make([]TransitionListener[StateT], 0), +// proposals: make(chan proposal[StateT], 1000), +// shouldEmitReceiveEventsOnSends: shouldEmitReceiveEventsOnSends, +// minimumProvers: minimumProvers, +// syncProvider: syncProvider, +// votingProvider: votingProvider, +// leaderProvider: leaderProvider, +// livenessProvider: livenessProvider, +// pacemakerProvider: pacemakerProvider, +// latestTimeoutCertificate: &atomic.Value{}, +// latestQuorumCertificate: &atomic.Value{}, +// latestPartialTimeoutCertificate: &atomic.Value{}, +// timeoutCertificateCh: make(chan models.TimeoutCertificate), +// quorumCertificateCh: make(chan models.QuorumCertificate), +// partialTimeoutCertificateCh: make(chan models.TimeoutCertificate), +// traceLogger: traceLogger, +// } + +// // // Define state configurations +// // sm.defineStateConfigs() + +// // // Define transitions +// // sm.defineTransitions() + +// // // Start event processor +// // go sm.processEvents() + +// return sm +// } + +// // Start begins the state machine +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) Start() error { +// sm.traceLogger.Trace("enter start") +// defer sm.traceLogger.Trace("exit start") +// select { +// case <-sm.ctx.Done(): +// return nil +// case <-time.After(time.Until(sm.startTime)): +// sm.traceLogger.Trace("starting state machine") +// err := sm.runLoop(sm.ctx) +// if err != nil { +// sm.traceLogger.Error("error in run loop", err) +// return err +// } +// } +// return nil +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) runLoop(ctx context.Context) error { +// err := sm.pacemakerProvider.Start(ctx) +// if err != nil { +// return fmt.Errorf("could not start event handler: %w", err) +// } + +// for { +// timeoutChannel := sm.pacemakerProvider.TimeoutCh() + +// // the first select makes sure we process timeouts with priority +// select { + +// // if we receive the shutdown signal, exit the loop +// case <-ctx.Done(): +// return nil + +// // processing timeout or partial TC event are top priority since +// // they allow node to contribute to TC aggregation when replicas can't +// // make progress on happy path +// case <-timeoutChannel: +// processStart := time.Now() +// curRank := e.paceMaker.CurRank() +// e.log.Debug().Uint64("cur_rank", curRank).Msg("timeout received from event loop") +// e.notifier.OnLocalTimeout(curRank) +// defer e.notifier.OnEventProcessed() + +// err := e.broadcastTimeoutStateIfAuthorized() +// if err != nil { +// return fmt.Errorf("unexpected exception while processing timeout in rank %d: %w", curRank, err) +// } + +// // At this point, we have received and processed an event from the timeout channel. +// // A timeout also means that we have made progress. A new timeout will have +// // been started and el.eventHandler.TimeoutChannel() will be a NEW channel (for the just-started timeout). +// // Very important to start the for loop from the beginning, to continue the with the new timeout channel! +// continue + +// case <-partialTCs: + +// processStart := time.Now() +// err = el.eventHandler.OnPartialTimeoutCertificateCreated(el.newestSubmittedPartialTimeoutCertificate.NewestPartialTimeoutCertificate()) +// if err != nil { +// return fmt.Errorf("could no process partial created TC event: %w", err) +// } + +// // At this point, we have received and processed partial TC event, it could have resulted in several scenarios: +// // 1. a rank change with potential voting or proposal creation +// // 2. a created and broadcast timeout state +// // 3. QC and TC didn't result in rank change and no timeout was created since we have already timed out or +// // the partial TC was created for rank different from current one. +// continue + +// default: +// // fall through to non-priority events +// } + +// idleStart := time.Now() + +// // select for state headers/QCs here +// select { + +// // same as before +// case <-shutdownSignaled: +// return nil + +// // same as before +// case <-timeoutChannel: +// err = el.eventHandler.OnLocalTimeout() +// if err != nil { +// return fmt.Errorf("could not process timeout: %w", err) +// } + +// // if we have a new proposal, process it +// case queuedItem := <-el.proposals: +// processStart := time.Now() +// proposal := queuedItem.proposal +// err = el.eventHandler.OnReceiveProposal(proposal) +// if err != nil { +// return fmt.Errorf("could not process proposal %v: %w", proposal.State.Identifier, err) +// } + +// el.log.Info(). +// Dur("dur_ms", time.Since(processStart)). +// Uint64("rank", proposal.State.Rank). +// Hex("state_id", proposal.State.Identifier[:]). +// Msg("state proposal has been processed successfully") + +// // if we have a new QC, process it +// case <-quorumCertificates: +// processStart := time.Now() +// err = el.eventHandler.OnReceiveQuorumCertificate(el.newestSubmittedQc.NewestQC()) +// if err != nil { +// return fmt.Errorf("could not process QC: %w", err) +// } + +// // if we have a new TC, process it +// case <-timeoutCertificates: +// // measure how long the event loop was idle waiting for an +// // incoming event +// el.metrics.HotStuffIdleDuration(time.Since(idleStart)) + +// processStart := time.Now() +// err = el.eventHandler.OnReceiveTimeoutCertificate(el.newestSubmittedTimeoutCertificate.NewestTC()) +// if err != nil { +// return fmt.Errorf("could not process TC: %w", err) +// } + +// case <-partialTCs: +// // measure how long the event loop was idle waiting for an +// // incoming event +// el.metrics.HotStuffIdleDuration(time.Since(idleStart)) + +// processStart := time.Now() +// err = el.eventHandler.OnPartialTimeoutCertificateCreated(el.newestSubmittedPartialTimeoutCertificate.NewestPartialTimeoutCertificate()) +// if err != nil { +// return fmt.Errorf("could no process partial created TC event: %w", err) +// } +// } +// } +// } + +// // Stop halts the state machine +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) Stop() error { +// sm.traceLogger.Trace("enter stop") +// defer sm.traceLogger.Trace("exit stop") +// sm.cancel() +// return nil +// } + +// // ReceiveLivenessCheck receives a liveness announcement and captures +// // collected mutation operations reported by the peer if relevant. +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveLivenessCheck(peer PeerIDT, collected CollectedT) error { +// sm.traceLogger.Trace( +// fmt.Sprintf( +// "enter receivelivenesscheck, peer: %s, rank: %d", +// peer.Identity(), +// collected.GetRank(), +// ), +// ) +// defer sm.traceLogger.Trace("exit receivelivenesscheck") +// sm.mu.Lock() +// if _, ok := sm.liveness[collected.GetRank()]; !ok { +// sm.liveness[collected.GetRank()] = make(map[Identity]CollectedT) +// } +// if _, ok := sm.liveness[collected.GetRank()][peer.Identity()]; !ok { +// sm.liveness[collected.GetRank()][peer.Identity()] = collected +// } +// sm.mu.Unlock() + +// sm.SendEvent(EventLivenessCheckReceived) +// return nil +// } + +// // ReceiveProposal receives a proposed new state. +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveProposal(currentRank uint64, peer PeerIDT, proposal *StateT) error { +// sm.traceLogger.Trace("enter receiveproposal") +// defer sm.traceLogger.Trace("exit receiveproposal") +// sm.mu.Lock() +// if _, ok := sm.proposals[(*proposal).GetRank()]; !ok { +// sm.proposals[(*proposal).GetRank()] = make(map[Identity]*StateT) +// } +// if _, ok := sm.proposals[(*proposal).GetRank()][peer.Identity()]; !ok { +// sm.proposals[(*proposal).GetRank()][peer.Identity()] = proposal +// } +// sm.mu.Unlock() + +// sm.SendEvent(EventProposalReceived) +// return nil +// } + +// // ReceiveVote captures a vote. Presumes structural and protocol validity of a +// // vote has already been evaluated. +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveVote(proposer PeerIDT, voter PeerIDT, vote *VoteT) error { +// sm.traceLogger.Trace("enter receivevote") +// defer sm.traceLogger.Trace("exit receivevote") +// sm.mu.Lock() + +// if _, ok := sm.votes[(*vote).GetRank()]; !ok { +// sm.votes[(*vote).GetRank()] = make(map[Identity]*VoteT) +// } +// if _, ok := sm.votes[(*vote).GetRank()][voter.Identity()]; !ok { +// sm.votes[(*vote).GetRank()][voter.Identity()] = vote +// } else if (*sm.votes[(*vote).GetRank()][voter.Identity()]).Identity() != +// (*vote).Identity() { +// sm.mu.Unlock() +// return errors.Wrap(errors.New("received conflicting vote"), "receive vote") +// } +// sm.mu.Unlock() + +// sm.SendEvent(EventVoteReceived) +// return nil +// } + +// // ReceiveConfirmation captures a confirmation. Presumes structural and protocol +// // validity of the state has already been evaluated. +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveConfirmation( +// peer PeerIDT, +// confirmation *StateT, +// ) error { +// sm.traceLogger.Trace("enter receiveconfirmation") +// defer sm.traceLogger.Trace("exit receiveconfirmation") +// sm.mu.Lock() +// if _, ok := sm.confirmations[(*confirmation).GetRank()]; !ok { +// sm.confirmations[(*confirmation).GetRank()] = make(map[Identity]*StateT) +// } +// if _, ok := sm.confirmations[(*confirmation).GetRank()][peer.Identity()]; !ok { +// sm.confirmations[(*confirmation).GetRank()][peer.Identity()] = confirmation +// } +// sm.mu.Unlock() + +// sm.SendEvent(EventConfirmationReceived) +// return nil +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveInvalidProposal(peer PeerIDT, proposal *StateT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveEquivocatingProposals( +// peer PeerIDT, +// proposal1 *StateT, +// proposal2 *StateT, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveInvalidVote(peer PeerIDT, vote *VoteT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveEquivocatingVotes(peer PeerIDT, vote1 *VoteT, vote2 *VoteT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveVoteForInvalidProposal(peer PeerIDT, proposal *StateT, vote *VoteT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveInvalidTimeout(peer PeerIDT, timeout *models.TimeoutState) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveEquivocatingTimeout( +// peer PeerIDT, +// timeout1 *models.TimeoutState, +// timeout2 *models.TimeoutState, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveState(peer PeerIDT, state *StateT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveFinalizedState(peer PeerIDT, finalized *StateT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) EventProcessed() { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) Started(currentRank uint64) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveQuorumCertificate( +// currentRank uint64, +// peer PeerIDT, +// cert models.QuorumCertificate, +// ) error { +// return nil +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveTimeoutCertificate( +// currentRank uint64, +// peer PeerIDT, +// cert models.TimeoutCertificate, +// ) error { +// return nil +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveLocalTimeout(currentRank uint64) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveNewRank(oldRank uint64, newRank uint64) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveQuorumCertificateWithNewRank( +// oldRank uint64, +// newRank uint64, +// cert models.QuorumCertificate, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveTimeoutCertificateWithNewRank( +// oldRank uint64, +// newRank uint64, +// cert models.TimeoutCertificate, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveNewTimeout(start time.Time, end time.Time) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveAggregatedQuorumCertificate( +// peer PeerIDT, +// cert models.QuorumCertificate, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) VoteProcessed(peer PeerIDT, vote *VoteT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveAggregatedTimeoutCertificate( +// peer PeerIDT, +// cert models.TimeoutCertificate, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveIncompleteTimeoutCertificate( +// rank uint64, +// peer PeerIDT, +// latestQuorumCert models.QuorumCertificate, +// previousRankTimeoutCert models.TimeoutCertificate, +// ) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveNewQuorumCertificateFromTimeout(cert models.QuorumCertificate) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) ReceiveNewTimeoutCertificateFromTimeout(cert models.TimeoutCertificate) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) TimeoutProcessed(timeout *models.TimeoutState) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) SendingVote(vote *VoteT) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) SendingTimeout(timeout *models.TimeoutState) { +// } + +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) SendingProposal(proposal *StateT, target time.Time) { +// } + +// // // SendEvent sends an event to the state machine +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) SendEvent(event Event) { +// // sm.traceLogger.Trace(fmt.Sprintf("enter sendEvent: %s", event)) +// // defer sm.traceLogger.Trace(fmt.Sprintf("exit sendEvent: %s", event)) +// // response := make(chan error, 1) +// // go func() { +// // select { +// // case sm.eventChan <- eventWrapper{event: event, response: response}: +// // <-response +// // case <-sm.ctx.Done(): +// // return +// // } +// // }() +// // } + +// // // processEvents handles events and transitions +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) processEvents() { +// // defer func() { +// // if r := recover(); r != nil { +// // sm.traceLogger.Error( +// // "fatal error encountered", +// // errors.New(fmt.Sprintf("%+v", r)), +// // ) +// // sm.Close() +// // } +// // }() + +// // sm.traceLogger.Trace("enter processEvents") +// // defer sm.traceLogger.Trace("exit processEvents") +// // for { +// // select { +// // case <-sm.ctx.Done(): +// // return +// // case wrapper := <-sm.eventChan: +// // err := sm.handleEvent(wrapper.event) +// // wrapper.response <- err +// // } +// // } +// // } + +// // // handleEvent processes a single event +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) handleEvent(event Event) error { +// // sm.traceLogger.Trace(fmt.Sprintf("enter handleEvent: %s", event)) +// // defer sm.traceLogger.Trace(fmt.Sprintf("exit handleEvent: %s", event)) +// // sm.mu.Lock() + +// // currentState := sm.machineState +// // transitions, exists := sm.transitions[currentState] +// // if !exists { +// // sm.mu.Unlock() + +// // return errors.Wrap( +// // fmt.Errorf("no transitions defined for state %s", currentState), +// // "handle event", +// // ) +// // } + +// // transition, exists := transitions[event] +// // if !exists { +// // sm.mu.Unlock() + +// // return errors.Wrap( +// // fmt.Errorf( +// // "no transition for event %s in state %s", +// // event, +// // currentState, +// // ), +// // "handle event", +// // ) +// // } + +// // // Check guard condition with the actual state +// // if transition.Guard != nil && !transition.Guard(sm) { +// // sm.mu.Unlock() + +// // return errors.Wrap( +// // fmt.Errorf( +// // "transition guard failed for %s -> %s on %s", +// // currentState, +// // transition.To, +// // event, +// // ), +// // "handle event", +// // ) +// // } + +// // sm.mu.Unlock() + +// // // Execute transition +// // sm.executeTransition(currentState, transition.To, event) +// // return nil +// // } + +// // // executeTransition performs the state transition +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) executeTransition( +// // from State, +// // to State, +// // event Event, +// // ) { +// // sm.traceLogger.Trace( +// // fmt.Sprintf("enter executeTransition: %s -> %s [%s]", from, to, event), +// // ) +// // defer sm.traceLogger.Trace( +// // fmt.Sprintf("exit executeTransition: %s -> %s [%s]", from, to, event), +// // ) +// // sm.mu.Lock() + +// // // Cancel any existing timeout and behavior +// // if sm.timeoutTimer != nil { +// // sm.timeoutTimer.Stop() +// // sm.timeoutTimer = nil +// // } + +// // // Cancel existing behavior if any +// // if sm.behaviorCancel != nil { +// // sm.behaviorCancel() +// // sm.behaviorCancel = nil +// // } + +// // // Call exit callback for current state +// // if config, exists := sm.stateConfigs[from]; exists && config.OnExit != nil { +// // sm.mu.Unlock() +// // config.OnExit(sm, sm.activeState, event) +// // sm.mu.Lock() +// // } + +// // // Update state +// // sm.machineState = to +// // sm.stateStartTime = time.Now() +// // sm.transitionCount++ + +// // // Notify listeners +// // for _, listener := range sm.listeners { +// // listener.OnTransition(from, to, event) +// // } + +// // // Call enter callback for new state +// // if config, exists := sm.stateConfigs[to]; exists { +// // if config.OnEnter != nil { +// // sm.mu.Unlock() +// // config.OnEnter(sm, sm.activeState, event) +// // sm.mu.Lock() +// // } + +// // // Start state behavior if defined +// // if config.Behavior != nil { +// // behaviorCtx, cancel := context.WithCancel(sm.ctx) +// // sm.behaviorCancel = cancel +// // sm.mu.Unlock() +// // config.Behavior(sm, sm.activeState, behaviorCtx) +// // sm.mu.Lock() +// // } + +// // // Set up timeout for new state +// // if config.Timeout > 0 && config.OnTimeout != "" { +// // sm.timeoutTimer = time.AfterFunc(config.Timeout, func() { +// // sm.SendEvent(config.OnTimeout) +// // }) +// // } +// // } +// // sm.mu.Unlock() +// // } + +// // // GetState returns the current state +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) GetState() State { +// // sm.traceLogger.Trace("enter getstate") +// // defer sm.traceLogger.Trace("exit getstate") +// // sm.mu.Lock() +// // defer sm.mu.Unlock() +// // return sm.machineState +// // } + +// // // Additional methods for compatibility +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) GetStateTime() time.Duration { +// // sm.traceLogger.Trace("enter getstatetime") +// // defer sm.traceLogger.Trace("exit getstatetime") +// // return time.Since(sm.stateStartTime) +// // } + +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) GetTransitionCount() uint64 { +// // sm.traceLogger.Trace("enter transitioncount") +// // defer sm.traceLogger.Trace("exit transitioncount") +// // return sm.transitionCount +// // } + +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) AddListener(listener TransitionListener[StateT]) { +// // sm.traceLogger.Trace("enter addlistener") +// // defer sm.traceLogger.Trace("exit addlistener") +// // sm.mu.Lock() +// // defer sm.mu.Unlock() +// // sm.listeners = append(sm.listeners, listener) +// // } + +// // func (sm *StateMachine[ +// // StateT, +// // VoteT, +// // PeerIDT, +// // CollectedT, +// // ]) Close() { +// // sm.traceLogger.Trace("enter close") +// // defer sm.traceLogger.Trace("exit close") +// // sm.mu.Lock() +// // defer sm.mu.Unlock() +// // sm.cancel() +// // if sm.timeoutTimer != nil { +// // sm.timeoutTimer.Stop() +// // } +// // if sm.behaviorCancel != nil { +// // sm.behaviorCancel() +// // } +// // sm.machineState = StateStopped +// // } + +// // Type used to satisfy generic arguments in compiler time type assertion check +// type nilUnique struct{} + +// // GetTimestamp implements models.Unique. +// func (n *nilUnique) GetTimestamp() uint64 { +// panic("unimplemented") +// } + +// // Source implements models.Unique. +// func (n *nilUnique) Source() models.Identity { +// panic("unimplemented") +// } + +// // Clone implements models.Unique. +// func (n *nilUnique) Clone() models.Unique { +// panic("unimplemented") +// } + +// // GetRank implements models.Unique. +// func (n *nilUnique) GetRank() uint64 { +// panic("unimplemented") +// } + +// // Identity implements models.Unique. +// func (n *nilUnique) Identity() models.Identity { +// panic("unimplemented") +// } + +// var _ models.Unique = (*nilUnique)(nil) + +// /* +// // defineStateConfigs sets up state configurations with behaviors +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) defineStateConfigs() { +// sm.traceLogger.Trace("enter defineStateConfigs") +// defer sm.traceLogger.Trace("exit defineStateConfigs") +// // Starting state - just timeout to complete initialization +// sm.stateConfigs[StateStarting] = &StateConfig[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]{ +// Timeout: 1 * time.Second, +// OnTimeout: EventInitComplete, +// } + +// type Config = StateConfig[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ] + +// type SMT = StateMachine[StateT, VoteT, PeerIDT, CollectedT] + +// // Loading state - synchronize with network +// sm.stateConfigs[StateLoading] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Loading behavior") +// defer sm.traceLogger.Trace("exit Loading behavior") +// if sm.syncProvider != nil { +// newStateCh, errCh := sm.syncProvider.Synchronize(ctx, sm.activeState) +// select { +// case newState := <-newStateCh: +// sm.mu.Lock() +// sm.activeState = newState +// sm.mu.Unlock() +// nextLeaders, err := sm.leaderProvider.GetNextLeaders(ctx, newState) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// time.Sleep(10 * time.Second) +// sm.SendEvent(EventSyncTimeout) +// return +// } +// found := false +// for _, leader := range nextLeaders { +// if leader.Identity() == sm.id.Identity() { +// found = true +// break +// } +// } +// if found { +// sm.SendEvent(EventSyncComplete) +// } else { +// time.Sleep(10 * time.Second) +// sm.SendEvent(EventSyncTimeout) +// } +// case <-errCh: +// time.Sleep(10 * time.Second) +// sm.SendEvent(EventSyncTimeout) +// case <-ctx.Done(): +// return +// } +// } +// }, +// Timeout: 10 * time.Hour, +// OnTimeout: EventSyncTimeout, +// } + +// // Collecting state - wait for frame or timeout +// sm.stateConfigs[StateCollecting] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Collecting behavior") +// defer sm.traceLogger.Trace("exit Collecting behavior") +// collected, err := sm.livenessProvider.Collect(ctx) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } + +// sm.mu.Lock() +// sm.nextProvers = []PeerIDT{} +// sm.chosenProposer = nil +// sm.collected = &collected +// sm.mu.Unlock() + +// nextProvers, err := sm.leaderProvider.GetNextLeaders(ctx, data) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } + +// sm.mu.Lock() +// sm.nextProvers = nextProvers +// sm.mu.Unlock() + +// err = sm.livenessProvider.SendLiveness(ctx, data, collected) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } + +// sm.mu.Lock() +// if sm.shouldEmitReceiveEventsOnSends { +// if _, ok := sm.liveness[collected.GetRank()]; !ok { +// sm.liveness[collected.GetRank()] = make(map[Identity]CollectedT) +// } +// sm.liveness[collected.GetRank()][sm.id.Identity()] = *sm.collected +// } +// sm.mu.Unlock() + +// if sm.shouldEmitReceiveEventsOnSends { +// sm.SendEvent(EventLivenessCheckReceived) +// } + +// sm.SendEvent(EventCollectionDone) +// }, +// Timeout: 10 * time.Second, +// OnTimeout: EventInduceSync, +// } + +// // Liveness check state +// sm.stateConfigs[StateLivenessCheck] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Liveness behavior") +// defer sm.traceLogger.Trace("exit Liveness behavior") +// sm.mu.Lock() +// nextProversLen := len(sm.nextProvers) +// sm.mu.Unlock() + +// // If we're not meeting the minimum prover count, we should loop. +// if nextProversLen < int(sm.minimumProvers()) { +// sm.traceLogger.Trace("insufficient provers, re-fetching leaders") +// var err error +// nextProvers, err := sm.leaderProvider.GetNextLeaders(ctx, data) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } +// sm.mu.Lock() +// sm.nextProvers = nextProvers +// sm.mu.Unlock() +// } + +// sm.mu.Lock() +// collected := *sm.collected +// sm.mu.Unlock() + +// sm.mu.Lock() +// livenessLen := len(sm.liveness[(*sm.activeState).GetRank()+1]) +// sm.mu.Unlock() + +// // We have enough checks for consensus: +// if livenessLen >= int(sm.minimumProvers()) { +// sm.traceLogger.Trace( +// "sufficient liveness checks, sending prover signal", +// ) +// sm.SendEvent(EventProverSignal) +// return +// } + +// sm.traceLogger.Trace( +// fmt.Sprintf( +// "insufficient liveness checks: need %d, have %d", +// sm.minimumProvers(), +// livenessLen, +// ), +// ) + +// select { +// case <-time.After(1 * time.Second): +// err := sm.livenessProvider.SendLiveness(ctx, data, collected) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } +// case <-ctx.Done(): +// } +// }, +// Timeout: 2 * time.Second, +// OnTimeout: EventLivenessTimeout, +// } + +// // Proving state - generate proof +// sm.stateConfigs[StateProving] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Proving behavior") +// defer sm.traceLogger.Trace("exit Proving behavior") +// sm.mu.Lock() +// collected := sm.collected +// sm.collected = nil +// sm.mu.Unlock() + +// if collected == nil { +// sm.SendEvent(EventInduceSync) +// return +// } + +// proposal, err := sm.leaderProvider.ProveNextState( +// ctx, +// data, +// *collected, +// ) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) + +// sm.SendEvent(EventInduceSync) +// return +// } + +// sm.mu.Lock() +// sm.traceLogger.Trace( +// fmt.Sprintf("adding proposal with rank %d", (*proposal).GetRank()), +// ) +// if _, ok := sm.proposals[(*proposal).GetRank()]; !ok { +// sm.proposals[(*proposal).GetRank()] = make(map[Identity]*StateT) +// } +// sm.proposals[(*proposal).GetRank()][sm.id.Identity()] = proposal +// sm.mu.Unlock() + +// sm.SendEvent(EventProofComplete) +// }, +// Timeout: 120 * time.Second, +// OnTimeout: EventPublishTimeout, +// } + +// // Publishing state - publish frame +// sm.stateConfigs[StatePublishing] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Publishing behavior") +// defer sm.traceLogger.Trace("exit Publishing behavior") +// sm.mu.Lock() +// if _, ok := sm.proposals[(*data).GetRank()+1][sm.id.Identity()]; ok { +// proposal := sm.proposals[(*data).GetRank()+1][sm.id.Identity()] +// sm.mu.Unlock() + +// err := sm.votingProvider.SendProposal( +// ctx, +// proposal, +// ) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } +// sm.SendEvent(EventPublishComplete) +// } else { +// sm.mu.Unlock() +// } +// }, +// Timeout: 1 * time.Second, +// OnTimeout: EventPublishTimeout, +// } + +// // Voting state - monitor for quorum +// sm.stateConfigs[StateVoting] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Voting behavior") +// defer sm.traceLogger.Trace("exit Voting behavior") + +// sm.mu.Lock() + +// if sm.chosenProposer == nil { +// // We haven't voted yet +// sm.traceLogger.Trace("proposer not yet chosen") +// perfect := map[int]PeerIDT{} // all provers +// live := map[int]PeerIDT{} // the provers who told us they're alive +// for i, p := range sm.nextProvers { +// perfect[i] = p +// if _, ok := sm.liveness[(*sm.activeState).GetRank()+1][p.Identity()]; ok { +// live[i] = p +// } +// } + +// if len(sm.proposals[(*sm.activeState).GetRank()+1]) < int(sm.minimumProvers()) { +// sm.traceLogger.Trace( +// fmt.Sprintf( +// "insufficient proposal count: %d, need %d", +// len(sm.proposals[(*sm.activeState).GetRank()+1]), +// int(sm.minimumProvers()), +// ), +// ) +// sm.mu.Unlock() +// return +// } + +// if ctx == nil { +// sm.traceLogger.Trace("context null") +// sm.mu.Unlock() +// return +// } + +// select { +// case <-ctx.Done(): +// sm.traceLogger.Trace("context canceled") +// sm.mu.Unlock() +// return +// default: +// sm.traceLogger.Trace("choosing proposal") +// proposals := map[Identity]*StateT{} +// for k, v := range sm.proposals[(*sm.activeState).GetRank()+1] { +// state := (*v).Clone().(StateT) +// proposals[k] = &state +// } + +// sm.mu.Unlock() +// selectedPeer, vote, err := sm.votingProvider.DecideAndSendVote( +// ctx, +// proposals, +// ) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// break +// } +// sm.mu.Lock() +// sm.chosenProposer = &selectedPeer + +// if sm.shouldEmitReceiveEventsOnSends { +// if _, ok := sm.votes[(*sm.activeState).GetRank()+1]; !ok { +// sm.votes[(*sm.activeState).GetRank()+1] = make(map[Identity]*VoteT) +// } +// sm.votes[(*sm.activeState).GetRank()+1][sm.id.Identity()] = vote +// sm.mu.Unlock() +// sm.SendEvent(EventVoteReceived) +// return +// } +// sm.mu.Unlock() +// } +// } else { +// sm.traceLogger.Trace("proposal chosen, checking for quorum") +// proposalVotes := map[Identity]*VoteT{} +// for p, vp := range sm.votes[(*sm.activeState).GetRank()+1] { +// vclone := (*vp).Clone().(VoteT) +// proposalVotes[p] = &vclone +// } +// haveEnoughProposals := len(sm.proposals[(*sm.activeState).GetRank()+1]) >= +// int(sm.minimumProvers()) +// sm.mu.Unlock() +// isQuorum, err := sm.votingProvider.IsQuorum(ctx, proposalVotes) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } + +// if isQuorum && haveEnoughProposals { +// sm.traceLogger.Trace("quorum reached") +// sm.SendEvent(EventQuorumReached) +// } else { +// sm.traceLogger.Trace( +// fmt.Sprintf( +// "quorum not reached: proposals: %d, needed: %d", +// len(sm.proposals[(*sm.activeState).GetRank()+1]), +// sm.minimumProvers(), +// ), +// ) +// } +// } +// }, +// Timeout: 1 * time.Second, +// OnTimeout: EventVotingTimeout, +// } + +// // Finalizing state +// sm.stateConfigs[StateFinalizing] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.mu.Lock() +// proposals := map[Identity]*StateT{} +// for k, v := range sm.proposals[(*sm.activeState).GetRank()+1] { +// state := (*v).Clone().(StateT) +// proposals[k] = &state +// } +// proposalVotes := map[Identity]*VoteT{} +// for p, vp := range sm.votes[(*sm.activeState).GetRank()+1] { +// vclone := (*vp).Clone().(VoteT) +// proposalVotes[p] = &vclone +// } +// sm.mu.Unlock() +// finalized, _, err := sm.votingProvider.FinalizeVotes( +// ctx, +// proposals, +// proposalVotes, +// ) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } +// next := (*finalized).Clone().(StateT) +// sm.mu.Lock() +// sm.nextState = &next +// sm.mu.Unlock() +// sm.SendEvent(EventAggregationDone) +// }, +// Timeout: 1 * time.Second, +// OnTimeout: EventAggregationTimeout, +// } + +// // Verifying state +// sm.stateConfigs[StateVerifying] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.traceLogger.Trace("enter Verifying behavior") +// defer sm.traceLogger.Trace("exit Verifying behavior") +// sm.mu.Lock() +// if _, ok := sm.confirmations[(*sm.activeState).GetRank()+1][sm.id.Identity()]; !ok && +// sm.nextState != nil { +// nextState := sm.nextState +// sm.mu.Unlock() +// err := sm.votingProvider.SendConfirmation(ctx, nextState) +// if err != nil { +// sm.traceLogger.Error( +// fmt.Sprintf("error encountered in %s", sm.machineState), +// err, +// ) +// sm.SendEvent(EventInduceSync) +// return +// } +// sm.mu.Lock() +// } + +// progressed := false +// if sm.nextState != nil { +// sm.activeState = sm.nextState +// progressed = true +// } +// if progressed { +// sm.nextState = nil +// sm.collected = nil +// delete(sm.liveness, (*sm.activeState).GetRank()) +// delete(sm.proposals, (*sm.activeState).GetRank()) +// delete(sm.votes, (*sm.activeState).GetRank()) +// delete(sm.confirmations, (*sm.activeState).GetRank()) +// sm.mu.Unlock() +// sm.SendEvent(EventVerificationDone) +// } else { +// sm.mu.Unlock() +// } +// }, +// Timeout: 1 * time.Second, +// OnTimeout: EventVerificationTimeout, +// } + +// // Stopping state +// sm.stateConfigs[StateStopping] = &Config{ +// Behavior: func(sm *SMT, data *StateT, ctx context.Context) { +// sm.SendEvent(EventCleanupComplete) +// }, +// Timeout: 30 * time.Second, +// OnTimeout: EventCleanupComplete, +// } +// } + +// // defineTransitions sets up all possible state transitions +// func (sm *StateMachine[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) defineTransitions() { +// sm.traceLogger.Trace("enter defineTransitions") +// defer sm.traceLogger.Trace("exit defineTransitions") + +// // Helper to add transition +// addTransition := func( +// from State, +// event Event, +// to State, +// guard TransitionGuard[StateT, VoteT, PeerIDT, CollectedT], +// ) { +// if sm.transitions[from] == nil { +// sm.transitions[from] = make(map[Event]*Transition[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]) +// } +// sm.transitions[from][event] = &Transition[ +// StateT, +// VoteT, +// PeerIDT, +// CollectedT, +// ]{ +// From: from, +// Event: event, +// To: to, +// Guard: guard, +// } +// } + +// // Basic flow transitions +// addTransition(StateStopped, EventStart, StateStarting, nil) +// addTransition(StateStarting, EventInitComplete, StateLoading, nil) +// addTransition(StateLoading, EventSyncTimeout, StateLoading, nil) +// addTransition(StateLoading, EventSyncComplete, StateCollecting, nil) +// addTransition(StateCollecting, EventCollectionDone, StateLivenessCheck, nil) +// addTransition(StateLivenessCheck, EventProverSignal, StateProving, nil) + +// // Loop indefinitely if nobody can be found +// addTransition( +// StateLivenessCheck, +// EventLivenessTimeout, +// StateLivenessCheck, +// nil, +// ) +// // // Loop until we get enough of these +// // addTransition( +// // StateLivenessCheck, +// // EventLivenessCheckReceived, +// // StateLivenessCheck, +// // nil, +// // ) + +// // Prover flow +// addTransition(StateProving, EventProofComplete, StatePublishing, nil) +// addTransition(StateProving, EventPublishTimeout, StateVoting, nil) +// addTransition(StatePublishing, EventPublishComplete, StateVoting, nil) +// addTransition(StatePublishing, EventPublishTimeout, StateVoting, nil) + +// // Common voting flow +// addTransition(StateVoting, EventProposalReceived, StateVoting, nil) +// // addTransition(StateVoting, EventVoteReceived, StateVoting, nil) +// addTransition(StateVoting, EventQuorumReached, StateFinalizing, nil) +// addTransition(StateVoting, EventVotingTimeout, StateVoting, nil) +// addTransition(StateFinalizing, EventAggregationDone, StateVerifying, nil) +// addTransition(StateFinalizing, EventAggregationTimeout, StateFinalizing, nil) +// addTransition(StateVerifying, EventVerificationDone, StateCollecting, nil) +// addTransition(StateVerifying, EventVerificationTimeout, StateVerifying, nil) + +// // Stop or induce Sync transitions from any state +// for _, state := range []State{ +// StateStarting, +// StateLoading, +// StateCollecting, +// StateLivenessCheck, +// StateProving, +// StatePublishing, +// StateVoting, +// StateFinalizing, +// StateVerifying, +// } { +// addTransition(state, EventStop, StateStopping, nil) +// addTransition(state, EventInduceSync, StateLoading, nil) +// } + +// addTransition(StateStopping, EventCleanupComplete, StateStopped, nil) +// } +// */ diff --git a/consensus/timeoutaggregator/timeout_aggregator_test.go b/consensus/timeoutaggregator/timeout_aggregator_test.go new file mode 100644 index 0000000..95c541e --- /dev/null +++ b/consensus/timeoutaggregator/timeout_aggregator_test.go @@ -0,0 +1,133 @@ +package timeoutaggregator + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +func TestTimeoutAggregator(t *testing.T) { + suite.Run(t, new(TimeoutAggregatorTestSuite)) +} + +// TimeoutAggregatorTestSuite is a test suite for isolated testing of TimeoutAggregator. +// Contains mocked state which is used to verify correct behavior of TimeoutAggregator. +// Automatically starts and stops module.Startable in SetupTest and TearDownTest respectively. +type TimeoutAggregatorTestSuite struct { + suite.Suite + + lowestRetainedRank uint64 + highestKnownRank uint64 + aggregator *TimeoutAggregator[*helper.TestVote] + collectors *mocks.TimeoutCollectors[*helper.TestVote] + stopAggregator context.CancelFunc +} + +func (s *TimeoutAggregatorTestSuite) SetupTest() { + var err error + s.collectors = mocks.NewTimeoutCollectors[*helper.TestVote](s.T()) + + s.lowestRetainedRank = 100 + + s.aggregator, err = NewTimeoutAggregator( + helper.Logger(), + s.lowestRetainedRank, + s.collectors, + ) + require.NoError(s.T(), err) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := ctx + s.stopAggregator = cancel + s.aggregator.Start(signalerCtx) +} + +func (s *TimeoutAggregatorTestSuite) TearDownTest() { + s.stopAggregator() +} + +// TestAddTimeout_HappyPath tests a happy path when multiple threads are adding timeouts for processing +// Eventually every timeout has to be processed by TimeoutCollector +func (s *TimeoutAggregatorTestSuite) TestAddTimeout_HappyPath() { + timeoutsCount := 20 + collector := mocks.NewTimeoutCollector[*helper.TestVote](s.T()) + callCount := atomic.NewUint64(0) + collector.On("AddTimeout", mock.Anything).Run(func(mock.Arguments) { + callCount.Add(1) + }).Return(nil).Times(timeoutsCount) + s.collectors.On("GetOrCreateCollector", s.lowestRetainedRank).Return(collector, true, nil).Times(timeoutsCount) + + var start sync.WaitGroup + start.Add(timeoutsCount) + for i := 0; i < timeoutsCount; i++ { + go func() { + timeout := helper.TimeoutStateFixture[*helper.TestVote](helper.WithTimeoutStateRank[*helper.TestVote](s.lowestRetainedRank)) + + start.Done() + // Wait for last worker routine to signal ready. Then, + // feed all timeouts into cache + start.Wait() + + s.aggregator.AddTimeout(timeout) + }() + } + + start.Wait() + + require.Eventually(s.T(), func() bool { + return callCount.Load() == uint64(timeoutsCount) + }, time.Second, time.Millisecond*20) +} + +// TestAddTimeout_EpochUnknown tests if timeout states targeting unknown epoch should be ignored +func (s *TimeoutAggregatorTestSuite) TestAddTimeout_EpochUnknown() { + timeout := helper.TimeoutStateFixture(helper.WithTimeoutStateRank[*helper.TestVote](s.lowestRetainedRank)) + *s.collectors = *mocks.NewTimeoutCollectors[*helper.TestVote](s.T()) + done := make(chan struct{}) + s.collectors.On("GetOrCreateCollector", timeout.Rank).Return(nil, false, models.ErrRankUnknown).Run(func(args mock.Arguments) { + close(done) + }).Once() + s.aggregator.AddTimeout(timeout) + time.Sleep(100 * time.Millisecond) +} + +// TestPruneUpToRank tests that pruning removes collectors lower that retained rank +func (s *TimeoutAggregatorTestSuite) TestPruneUpToRank() { + s.collectors.On("PruneUpToRank", s.lowestRetainedRank+1).Once() + s.aggregator.PruneUpToRank(s.lowestRetainedRank + 1) +} + +// TestOnQuorumCertificateTriggeredRankChange tests if entering rank event gets processed when send through `TimeoutAggregator`. +// Tests the whole processing pipeline. +func (s *TimeoutAggregatorTestSuite) TestOnQuorumCertificateTriggeredRankChange() { + done := make(chan struct{}) + s.collectors.On("PruneUpToRank", s.lowestRetainedRank+1).Run(func(args mock.Arguments) { + close(done) + }).Once() + qc := helper.MakeQC(helper.WithQCRank(s.lowestRetainedRank)) + s.aggregator.OnRankChange(qc.GetRank(), qc.GetRank()+1) + time.Sleep(100 * time.Millisecond) +} + +// TestOnTimeoutCertificateTriggeredRankChange tests if entering rank event gets processed when send through `TimeoutAggregator`. +// Tests the whole processing pipeline. +func (s *TimeoutAggregatorTestSuite) TestOnTimeoutCertificateTriggeredRankChange() { + rank := s.lowestRetainedRank + 1 + done := make(chan struct{}) + s.collectors.On("PruneUpToRank", rank).Run(func(args mock.Arguments) { + close(done) + }).Once() + tc := helper.MakeTC(helper.WithTCRank(s.lowestRetainedRank)) + s.aggregator.OnRankChange(tc.GetRank(), tc.GetRank()+1) + time.Sleep(100 * time.Millisecond) +} diff --git a/consensus/timeoutaggregator/timeout_collectors_test.go b/consensus/timeoutaggregator/timeout_collectors_test.go new file mode 100644 index 0000000..5b8e663 --- /dev/null +++ b/consensus/timeoutaggregator/timeout_collectors_test.go @@ -0,0 +1,176 @@ +package timeoutaggregator + +import ( + "errors" + "fmt" + "sync" + "testing" + + "github.com/gammazero/workerpool" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +var factoryError = errors.New("factory error") + +func TestTimeoutCollectors(t *testing.T) { + suite.Run(t, new(TimeoutCollectorsTestSuite)) +} + +// TimeoutCollectorsTestSuite is a test suite for isolated testing of TimeoutCollectors. +// Contains helper methods and mocked state which is used to verify correct behavior of TimeoutCollectors. +type TimeoutCollectorsTestSuite struct { + suite.Suite + + mockedCollectors map[uint64]*mocks.TimeoutCollector[*helper.TestVote] + factoryMethod *mocks.TimeoutCollectorFactory[*helper.TestVote] + collectors *TimeoutCollectors[*helper.TestVote] + lowestRank uint64 + workerPool *workerpool.WorkerPool +} + +func (s *TimeoutCollectorsTestSuite) SetupTest() { + s.lowestRank = 1000 + s.mockedCollectors = make(map[uint64]*mocks.TimeoutCollector[*helper.TestVote]) + s.workerPool = workerpool.New(2) + s.factoryMethod = mocks.NewTimeoutCollectorFactory[*helper.TestVote](s.T()) + s.factoryMethod.On("Create", mock.Anything).Return(func(rank uint64) consensus.TimeoutCollector[*helper.TestVote] { + if collector, found := s.mockedCollectors[rank]; found { + return collector + } + return nil + }, func(rank uint64) error { + if _, found := s.mockedCollectors[rank]; found { + return nil + } + return fmt.Errorf("mocked collector %v not found: %w", rank, factoryError) + }).Maybe() + s.collectors = NewTimeoutCollectors(helper.Logger(), s.lowestRank, s.factoryMethod) +} + +func (s *TimeoutCollectorsTestSuite) TearDownTest() { + s.workerPool.StopWait() +} + +// prepareMockedCollector prepares a mocked collector and stores it in map, later it will be used +// to mock behavior of timeout collectors. +func (s *TimeoutCollectorsTestSuite) prepareMockedCollector(rank uint64) *mocks.TimeoutCollector[*helper.TestVote] { + collector := mocks.NewTimeoutCollector[*helper.TestVote](s.T()) + collector.On("Rank").Return(rank).Maybe() + s.mockedCollectors[rank] = collector + return collector +} + +// TestGetOrCreateCollector_RankLowerThanLowest tests a scenario where caller tries to create a collector with rank +// lower than already pruned one. This should result in sentinel error `BelowPrunedThresholdError` +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_RankLowerThanLowest() { + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestRank - 10) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.Error(s.T(), err) + require.True(s.T(), models.IsBelowPrunedThresholdError(err)) +} + +// TestGetOrCreateCollector_UnknownEpoch tests a scenario where caller tries to create a collector with rank referring epoch +// that we don't know about. This should result in sentinel error ` +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_UnknownEpoch() { + *s.factoryMethod = *mocks.NewTimeoutCollectorFactory[*helper.TestVote](s.T()) + s.factoryMethod.On("Create", mock.Anything).Return(nil, models.ErrRankUnknown) + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestRank + 100) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.ErrorIs(s.T(), err, models.ErrRankUnknown) +} + +// TestGetOrCreateCollector_ValidCollector tests a happy path scenario where we try first to create and then retrieve cached collector. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_ValidCollector() { + rank := s.lowestRank + 10 + s.prepareMockedCollector(rank) + collector, created, err := s.collectors.GetOrCreateCollector(rank) + require.NoError(s.T(), err) + require.True(s.T(), created) + require.Equal(s.T(), rank, collector.Rank()) + + cached, cachedCreated, err := s.collectors.GetOrCreateCollector(rank) + require.NoError(s.T(), err) + require.False(s.T(), cachedCreated) + require.Equal(s.T(), collector, cached) +} + +// TestGetOrCreateCollector_FactoryError tests that error from factory method is propagated to caller. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollector_FactoryError() { + // creating collector without calling prepareMockedCollector will yield factoryError. + collector, created, err := s.collectors.GetOrCreateCollector(s.lowestRank + 10) + require.Nil(s.T(), collector) + require.False(s.T(), created) + require.ErrorIs(s.T(), err, factoryError) +} + +// TestGetOrCreateCollectors_ConcurrentAccess tests that concurrently accessing of GetOrCreateCollector creates +// only one collector and all other instances are retrieved from cache. +func (s *TimeoutCollectorsTestSuite) TestGetOrCreateCollectors_ConcurrentAccess() { + createdTimes := atomic.NewUint64(0) + rank := s.lowestRank + 10 + s.prepareMockedCollector(rank) + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, created, err := s.collectors.GetOrCreateCollector(rank) + require.NoError(s.T(), err) + if created { + createdTimes.Add(1) + } + }() + } + wg.Wait() + + require.Equal(s.T(), uint64(1), createdTimes.Load()) +} + +// TestPruneUpToRank tests pruning removes item below pruning height and leaves unmodified other items. +func (s *TimeoutCollectorsTestSuite) TestPruneUpToRank() { + numberOfCollectors := uint64(10) + prunedRanks := make([]uint64, 0) + for i := uint64(0); i < numberOfCollectors; i++ { + rank := s.lowestRank + i + s.prepareMockedCollector(rank) + _, _, err := s.collectors.GetOrCreateCollector(rank) + require.NoError(s.T(), err) + prunedRanks = append(prunedRanks, rank) + } + + pruningHeight := s.lowestRank + numberOfCollectors + + expectedCollectors := make([]consensus.TimeoutCollector[*helper.TestVote], 0) + for i := uint64(0); i < numberOfCollectors; i++ { + rank := pruningHeight + i + s.prepareMockedCollector(rank) + collector, _, err := s.collectors.GetOrCreateCollector(rank) + require.NoError(s.T(), err) + expectedCollectors = append(expectedCollectors, collector) + } + + // after this operation collectors below pruning height should be pruned and everything higher + // should be left unmodified + s.collectors.PruneUpToRank(pruningHeight) + + for _, prunedRank := range prunedRanks { + _, _, err := s.collectors.GetOrCreateCollector(prunedRank) + require.Error(s.T(), err) + require.True(s.T(), models.IsBelowPrunedThresholdError(err)) + } + + for _, collector := range expectedCollectors { + cached, _, _ := s.collectors.GetOrCreateCollector(collector.Rank()) + require.Equal(s.T(), collector, cached) + } +} diff --git a/consensus/timeoutcollector/aggregation.go b/consensus/timeoutcollector/aggregation.go index 9e3b7db..04d1dce 100644 --- a/consensus/timeoutcollector/aggregation.go +++ b/consensus/timeoutcollector/aggregation.go @@ -177,7 +177,7 @@ func (a *TimeoutSignatureAggregator) Rank() uint64 { return a.rank } -// Aggregate aggregates the signatures and returns the aggregated signature. +// Aggregate aggregates the signatures and returns the aggregated consensus. // The resulting aggregated signature is guaranteed to be valid, as all // individual signatures are pre-validated before their addition. Expected // errors during normal operations: diff --git a/consensus/timeoutcollector/factory.go b/consensus/timeoutcollector/factory.go index 085695b..c201df9 100644 --- a/consensus/timeoutcollector/factory.go +++ b/consensus/timeoutcollector/factory.go @@ -64,6 +64,7 @@ type TimeoutProcessorFactory[ committee consensus.Replicas notifier consensus.TimeoutCollectorConsumer[VoteT] validator consensus.Validator[StateT, VoteT] + voting consensus.VotingProvider[StateT, VoteT, PeerIDT] domainSeparationTag []byte } @@ -127,6 +128,7 @@ func (f *TimeoutProcessorFactory[StateT, VoteT, PeerIDT]) Create(rank uint64) ( f.validator, sigAggregator, f.notifier, + f.voting, ) } diff --git a/consensus/timeoutcollector/timeout_cache_test.go b/consensus/timeoutcollector/timeout_cache_test.go new file mode 100644 index 0000000..7fcbceb --- /dev/null +++ b/consensus/timeoutcollector/timeout_cache_test.go @@ -0,0 +1,172 @@ +package timeoutcollector + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +// TestTimeoutStatesCache_Rank tests that Rank returns same value that was set by constructor +func TestTimeoutStatesCache_Rank(t *testing.T) { + rank := uint64(100) + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + require.Equal(t, rank, cache.Rank()) +} + +// TestTimeoutStatesCache_AddTimeoutStateRepeatedTimeout tests that AddTimeoutState skips duplicated timeouts +func TestTimeoutStatesCache_AddTimeoutStateRepeatedTimeout(t *testing.T) { + t.Parallel() + + rank := uint64(100) + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: rank, + }), + ) + + require.NoError(t, cache.AddTimeoutState(timeout)) + err := cache.AddTimeoutState(timeout) + require.ErrorIs(t, err, ErrRepeatedTimeout) + require.Len(t, cache.All(), 1) +} + +// TestTimeoutStatesCache_AddTimeoutStateIncompatibleRank tests that adding timeout with incompatible rank results in error +func TestTimeoutStatesCache_AddTimeoutStateIncompatibleRank(t *testing.T) { + t.Parallel() + + rank := uint64(100) + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank+1), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: rank, + }), + ) + err := cache.AddTimeoutState(timeout) + require.ErrorIs(t, err, ErrTimeoutForIncompatibleRank) +} + +// TestTimeoutStatesCache_GetTimeout tests that GetTimeout method returns the first added timeout +// for a given signer, if any timeout has been added. +func TestTimeoutStatesCache_GetTimeout(t *testing.T) { + rank := uint64(100) + knownTimeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: rank, + }), + ) + doubleTimeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: rank, + }), + ) + + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + + // unknown timeout + timeout, found := cache.GetTimeoutState(helper.MakeIdentity()) + require.Nil(t, timeout) + require.False(t, found) + + // known timeout + err := cache.AddTimeoutState(knownTimeout) + require.NoError(t, err) + timeout, found = cache.GetTimeoutState((*knownTimeout.Vote).ID) + require.Equal(t, knownTimeout, timeout) + require.True(t, found) + + // for a signer ID with a known timeout, the cache should memorize the _first_ encountered timeout + err = cache.AddTimeoutState(doubleTimeout) + require.True(t, models.IsDoubleTimeoutError[*helper.TestVote](err)) + timeout, found = cache.GetTimeoutState((*doubleTimeout.Vote).ID) + require.Equal(t, knownTimeout, timeout) + require.True(t, found) +} + +// TestTimeoutStatesCache_All tests that All returns previously added timeouts. +func TestTimeoutStatesCache_All(t *testing.T) { + t.Parallel() + + rank := uint64(100) + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + expectedTimeouts := make([]*models.TimeoutState[*helper.TestVote], 5) + for i := range expectedTimeouts { + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: fmt.Sprintf("%d", i), + Rank: rank, + }), + ) + expectedTimeouts[i] = timeout + require.NoError(t, cache.AddTimeoutState(timeout)) + } + require.ElementsMatch(t, expectedTimeouts, cache.All()) +} + +// BenchmarkAdd measured the time it takes to add `numberTimeouts` concurrently to the TimeoutStatesCache. +// On MacBook with Intel i7-7820HQ CPU @ 2.90GHz: +// adding 1 million timeouts in total, with 20 threads concurrently, took 0.48s +func BenchmarkAdd(b *testing.B) { + numberTimeouts := 1_000_000 + threads := 20 + + // Setup: create worker routines and timeouts to feed + rank := uint64(10) + cache := NewTimeoutStatesCache[*helper.TestVote](rank) + + var start sync.WaitGroup + start.Add(threads) + var done sync.WaitGroup + done.Add(threads) + + n := numberTimeouts / threads + + for ; threads > 0; threads-- { + go func(i int) { + // create timeouts and signal ready + timeouts := make([]models.TimeoutState[*helper.TestVote], 0, n) + for len(timeouts) < n { + t := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: rank, + }), + ) + timeouts = append(timeouts, *t) + } + + start.Done() + + // Wait for last worker routine to signal ready. Then, + // feed all timeouts into cache + start.Wait() + + for _, v := range timeouts { + err := cache.AddTimeoutState(&v) + require.NoError(b, err) + } + done.Done() + }(threads) + } + start.Wait() + t1 := time.Now() + done.Wait() + duration := time.Since(t1) + fmt.Printf("=> adding %d timeouts to Cache took %f seconds\n", cache.Size(), duration.Seconds()) +} diff --git a/consensus/timeoutcollector/timeout_collector_test.go b/consensus/timeoutcollector/timeout_collector_test.go new file mode 100644 index 0000000..9fe2f5b --- /dev/null +++ b/consensus/timeoutcollector/timeout_collector_test.go @@ -0,0 +1,230 @@ +package timeoutcollector + +import ( + "errors" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" +) + +func TestTimeoutCollector(t *testing.T) { + suite.Run(t, new(TimeoutCollectorTestSuite)) +} + +// TimeoutCollectorTestSuite is a test suite for testing TimeoutCollector. It stores mocked +// state internally for testing behavior. +type TimeoutCollectorTestSuite struct { + suite.Suite + + rank uint64 + notifier *mocks.TimeoutAggregationConsumer[*helper.TestVote] + processor *mocks.TimeoutProcessor[*helper.TestVote] + collector *TimeoutCollector[*helper.TestVote] +} + +func (s *TimeoutCollectorTestSuite) SetupTest() { + s.rank = 1000 + s.notifier = mocks.NewTimeoutAggregationConsumer[*helper.TestVote](s.T()) + s.processor = mocks.NewTimeoutProcessor[*helper.TestVote](s.T()) + + s.notifier.On("OnNewQuorumCertificateDiscovered", mock.Anything).Maybe() + s.notifier.On("OnNewTimeoutCertificateDiscovered", mock.Anything).Maybe() + + s.collector = NewTimeoutCollector(helper.Logger(), s.rank, s.notifier, s.processor) +} + +// TestRank tests that `Rank` returns the same value that was passed in constructor +func (s *TimeoutCollectorTestSuite) TestRank() { + require.Equal(s.T(), s.rank, s.collector.Rank()) +} + +// TestAddTimeout_HappyPath tests that process in happy path executed by multiple workers deliver expected results +// all operations should be successful, no errors expected +func (s *TimeoutCollectorTestSuite) TestAddTimeout_HappyPath() { + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: s.rank, + }), + ) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + }() + } + + s.processor.AssertExpectations(s.T()) +} + +// TestAddTimeout_DoubleTimeout tests that submitting two different timeouts for same rank ends with reporting +// double timeout to notifier which can be slashed later. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_DoubleTimeout() { + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: s.rank, + }), + ) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + + otherTimeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: "1", + Rank: s.rank, + }), + ) + + s.notifier.On("OnDoubleTimeoutDetected", timeout, otherTimeout).Once() + + err = s.collector.AddTimeout(otherTimeout) + require.NoError(s.T(), err) + s.notifier.AssertExpectations(s.T()) + s.processor.AssertNumberOfCalls(s.T(), "Process", 1) +} + +// TestAddTimeout_RepeatedTimeout checks that repeated timeouts are silently dropped without any errors. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_RepeatedTimeout() { + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: s.rank, + }), + ) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + err = s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + s.processor.AssertNumberOfCalls(s.T(), "Process", 1) +} + +// TestAddTimeout_TimeoutCacheException tests that submitting timeout state for rank which is not designated for this +// collector results in ErrTimeoutForIncompatibleRank. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_TimeoutCacheException() { + // incompatible rank is an exception and not handled by timeout collector + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank+1), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: s.rank + 1, + }), + ) + err := s.collector.AddTimeout(timeout) + require.ErrorIs(s.T(), err, ErrTimeoutForIncompatibleRank) + s.processor.AssertNotCalled(s.T(), "Process") +} + +// TestAddTimeout_InvalidTimeout tests that sentinel errors while processing timeouts are correctly handled and reported +// to notifier, but exceptions are propagated to caller. +func (s *TimeoutCollectorTestSuite) TestAddTimeout_InvalidTimeout() { + s.Run("invalid-timeout", func() { + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: s.rank, + }), + ) + s.processor.On("Process", timeout).Return(models.NewInvalidTimeoutErrorf(timeout, "")).Once() + s.notifier.On("OnInvalidTimeoutDetected", mock.Anything).Run(func(args mock.Arguments) { + invalidTimeoutErr := args.Get(0).(models.InvalidTimeoutError[*helper.TestVote]) + require.Equal(s.T(), timeout, invalidTimeoutErr.Timeout) + }).Once() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + + time.Sleep(100 * time.Millisecond) + s.notifier.AssertCalled(s.T(), "OnInvalidTimeoutDetected", mock.Anything) + }) + s.Run("process-exception", func() { + exception := errors.New("invalid-signature") + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutVote[*helper.TestVote](&helper.TestVote{ + ID: helper.MakeIdentity(), + Rank: s.rank, + }), + ) + s.processor.On("Process", timeout).Return(exception).Once() + err := s.collector.AddTimeout(timeout) + require.ErrorIs(s.T(), err, exception) + }) +} + +// TestAddTimeout_TONotifications tests that TimeoutCollector in happy path reports the newest discovered QC and TC +func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { + qcCount := 100 + // generate QCs with increasing rank numbers + if s.rank < uint64(qcCount) { + s.T().Fatal("invalid test configuration") + } + + *s.notifier = *mocks.NewTimeoutAggregationConsumer[*helper.TestVote](s.T()) + + var highestReportedQC models.QuorumCertificate + s.notifier.On("OnNewQuorumCertificateDiscovered", mock.Anything).Run(func(args mock.Arguments) { + qc := args.Get(0).(models.QuorumCertificate) + if highestReportedQC == nil || highestReportedQC.GetRank() < qc.GetRank() { + highestReportedQC = qc + } + }) + + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(s.rank - 1)) + s.notifier.On("OnNewTimeoutCertificateDiscovered", previousRankTimeoutCert).Once() + + timeouts := make([]*models.TimeoutState[*helper.TestVote], 0, qcCount) + for i := 0; i < qcCount; i++ { + qc := helper.MakeQC(helper.WithQCRank(uint64(i))) + timeout := helper.TimeoutStateFixture(func(timeout *models.TimeoutState[*helper.TestVote]) { + timeout.Rank = s.rank + timeout.LatestQuorumCertificate = qc + timeout.PriorRankTimeoutCertificate = previousRankTimeoutCert + }, helper.WithTimeoutVote(&helper.TestVote{Rank: s.rank, ID: helper.MakeIdentity()})) + timeouts = append(timeouts, timeout) + s.notifier.On("OnTimeoutProcessed", timeout).Once() + s.processor.On("Process", timeout).Return(nil).Once() + } + + expectedHighestQC := timeouts[len(timeouts)-1].LatestQuorumCertificate + + // shuffle timeouts in random order + rand.Shuffle(len(timeouts), func(i, j int) { + timeouts[i], timeouts[j] = timeouts[j], timeouts[i] + }) + + var wg sync.WaitGroup + wg.Add(len(timeouts)) + for _, timeout := range timeouts { + go func(timeout *models.TimeoutState[*helper.TestVote]) { + defer wg.Done() + err := s.collector.AddTimeout(timeout) + require.NoError(s.T(), err) + }(timeout) + } + wg.Wait() + + require.Equal(s.T(), expectedHighestQC, highestReportedQC) +} diff --git a/consensus/timeoutcollector/timeout_processor.go b/consensus/timeoutcollector/timeout_processor.go index cb4da65..ba51d8e 100644 --- a/consensus/timeoutcollector/timeout_processor.go +++ b/consensus/timeoutcollector/timeout_processor.go @@ -71,6 +71,7 @@ func NewTimeoutProcessor[ validator consensus.Validator[StateT, VoteT], sigAggregator consensus.TimeoutSignatureAggregator, notifier consensus.TimeoutCollectorConsumer[VoteT], + voting consensus.VotingProvider[StateT, VoteT, PeerIDT], ) (*TimeoutProcessor[StateT, VoteT, PeerIDT], error) { rank := sigAggregator.Rank() qcThreshold, err := committee.QuorumThresholdForRank(rank) @@ -105,6 +106,7 @@ func NewTimeoutProcessor[ }, sigAggregator: sigAggregator, newestQCTracker: tracker.NewNewestQCTracker(), + voting: voting, }, nil } @@ -159,7 +161,7 @@ func (p *TimeoutProcessor[StateT, VoteT, PeerIDT]) Process( p.newestQCTracker.Track(&timeout.LatestQuorumCertificate) totalWeight, err := p.sigAggregator.VerifyAndAdd( - (*timeout.Vote).Source(), + (*timeout.Vote).Identity(), (*timeout.Vote).GetSignature(), timeout.LatestQuorumCertificate.GetRank(), ) @@ -309,7 +311,7 @@ func (p *TimeoutProcessor[StateT, VoteT, PeerIDT]) validateTimeout( // 3. If TC is included, it must be valid if timeout.PriorRankTimeoutCertificate != nil { err = p.validator.ValidateTimeoutCertificate( - &timeout.PriorRankTimeoutCertificate, + timeout.PriorRankTimeoutCertificate, ) if err != nil { if models.IsInvalidTimeoutCertificateError(err) { diff --git a/consensus/timeoutcollector/timeout_processor_test.go b/consensus/timeoutcollector/timeout_processor_test.go new file mode 100644 index 0000000..891135b --- /dev/null +++ b/consensus/timeoutcollector/timeout_processor_test.go @@ -0,0 +1,678 @@ +package timeoutcollector + +import ( + "errors" + "fmt" + "math/rand" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.uber.org/atomic" + + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/helper" + "source.quilibrium.com/quilibrium/monorepo/consensus/mocks" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" + "source.quilibrium.com/quilibrium/monorepo/consensus/validator" + "source.quilibrium.com/quilibrium/monorepo/consensus/verification" + "source.quilibrium.com/quilibrium/monorepo/consensus/votecollector" +) + +func TestTimeoutProcessor(t *testing.T) { + suite.Run(t, new(TimeoutProcessorTestSuite)) +} + +// TimeoutProcessorTestSuite is a test suite that holds mocked state for isolated testing of TimeoutProcessor. +type TimeoutProcessorTestSuite struct { + suite.Suite + + participants []models.WeightedIdentity + signer models.WeightedIdentity + rank uint64 + sigWeight uint64 + totalWeight atomic.Uint64 + committee *mocks.Replicas + validator *mocks.Validator[*helper.TestState, *helper.TestVote] + sigAggregator *mocks.TimeoutSignatureAggregator + notifier *mocks.TimeoutCollectorConsumer[*helper.TestVote] + processor *TimeoutProcessor[*helper.TestState, *helper.TestVote, *helper.TestPeer] + voting *mocks.VotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer] +} + +func (s *TimeoutProcessorTestSuite) SetupTest() { + var err error + s.sigWeight = 1000 + s.committee = mocks.NewReplicas(s.T()) + s.validator = mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.sigAggregator = mocks.NewTimeoutSignatureAggregator(s.T()) + s.notifier = mocks.NewTimeoutCollectorConsumer[*helper.TestVote](s.T()) + s.participants = helper.WithWeightedIdentityList(11) + s.signer = s.participants[0] + s.rank = (uint64)(rand.Uint32() + 100) + s.totalWeight = *atomic.NewUint64(0) + s.voting = mocks.NewVotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer](s.T()) + + s.committee.On("QuorumThresholdForRank", mock.Anything).Return(uint64(8000), nil).Maybe() + s.committee.On("TimeoutThresholdForRank", mock.Anything).Return(uint64(8000), nil).Maybe() + s.committee.On("IdentityByEpoch", mock.Anything, mock.Anything).Return(s.signer, nil).Maybe() + s.sigAggregator.On("Rank").Return(s.rank).Maybe() + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + s.totalWeight.Add(s.sigWeight) + }).Return(func(signerID models.Identity, sig []byte, newestQCRank uint64) uint64 { + return s.totalWeight.Load() + }, func(signerID models.Identity, sig []byte, newestQCRank uint64) error { + return nil + }).Maybe() + s.sigAggregator.On("TotalWeight").Return(func() uint64 { + return s.totalWeight.Load() + }).Maybe() + + s.processor, err = NewTimeoutProcessor[*helper.TestState, *helper.TestVote, *helper.TestPeer]( + helper.Logger(), + s.committee, + s.validator, + s.sigAggregator, + s.notifier, + s.voting, + ) + require.NoError(s.T(), err) +} + +// TimeoutLastRankSuccessfulFixture creates a valid timeout if last rank has ended with QC. +func (s *TimeoutProcessorTestSuite) TimeoutLastRankSuccessfulFixture(opts ...func(*models.TimeoutState[*helper.TestVote])) *models.TimeoutState[*helper.TestVote] { + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutNewestQC[*helper.TestVote](helper.MakeQC(helper.WithQCRank(s.rank-1))), + helper.WithTimeoutVote(&helper.TestVote{ID: helper.MakeIdentity(), Rank: s.rank}), + helper.WithTimeoutPreviousRankTimeoutCertificate[*helper.TestVote](nil), + ) + + for _, opt := range opts { + opt(timeout) + } + + return timeout +} + +// TimeoutLastRankFailedFixture creates a valid timeout if last rank has ended with TC. +func (s *TimeoutProcessorTestSuite) TimeoutLastRankFailedFixture(opts ...func(*models.TimeoutState[*helper.TestVote])) *models.TimeoutState[*helper.TestVote] { + newestQC := helper.MakeQC(helper.WithQCRank(s.rank - 10)) + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutNewestQC[*helper.TestVote](newestQC), + helper.WithTimeoutVote(&helper.TestVote{ID: helper.MakeIdentity(), Rank: s.rank}), + helper.WithTimeoutPreviousRankTimeoutCertificate[*helper.TestVote](helper.MakeTC( + helper.WithTCRank(s.rank-1), + helper.WithTCNewestQC(helper.MakeQC(helper.WithQCRank(newestQC.GetRank()))))), + ) + + for _, opt := range opts { + opt(timeout) + } + + return timeout +} + +// TestProcess_TimeoutNotForRank tests that TimeoutProcessor accepts only timeouts for the rank it was initialized with +// We expect dedicated sentinel errors for timeouts for different ranks (`ErrTimeoutForIncompatibleRank`). +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutNotForRank() { + err := s.processor.Process(s.TimeoutLastRankSuccessfulFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.Rank++ + })) + require.ErrorIs(s.T(), err, ErrTimeoutForIncompatibleRank) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + + s.sigAggregator.AssertNotCalled(s.T(), "Verify") +} + +// TestProcess_TimeoutWithoutQC tests that TimeoutProcessor fails with models.InvalidTimeoutError if +// timeout doesn't contain QC. +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutWithoutQC() { + err := s.processor.Process(s.TimeoutLastRankSuccessfulFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.LatestQuorumCertificate = nil + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) +} + +// TestProcess_TimeoutNewerHighestQC tests that TimeoutProcessor fails with models.InvalidTimeoutError if +// timeout contains a QC with QC.Rank > timeout.Rank, QC can be only with lower rank than timeout. +func (s *TimeoutProcessorTestSuite) TestProcess_TimeoutNewerHighestQC() { + s.Run("t.Rank == t.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank", func() { + err := s.processor.Process(s.TimeoutLastRankSuccessfulFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank = t.Rank + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + }) + s.Run("t.Rank < t.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank", func() { + err := s.processor.Process(s.TimeoutLastRankSuccessfulFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank = t.Rank + 1 + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + }) +} + +// TestProcess_PreviousRankTimeoutCertificateWrongRank tests that TimeoutProcessor fails with models.InvalidTimeoutError if +// timeout contains a proof that sender legitimately entered timeout.Rank but it has wrong rank meaning he used TC from previous rounds. +func (s *TimeoutProcessorTestSuite) TestProcess_PreviousRankTimeoutCertificateWrongRank() { + // if TC is included it must have timeout.Rank == timeout.PriorRankTimeoutCertificate.(*helper.TestTimeoutCertificate).Rank+1 + err := s.processor.Process(s.TimeoutLastRankFailedFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.PriorRankTimeoutCertificate.(*helper.TestTimeoutCertificate).Rank = t.Rank - 10 + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) +} + +// TestProcess_LastRankHighestQCInvalidRank tests that TimeoutProcessor fails with models.InvalidTimeoutError if +// timeout contains a proof that sender legitimately entered timeout.Rank but included HighestQC has older rank +// than QC included in TC. For honest nodes this shouldn't happen. +func (s *TimeoutProcessorTestSuite) TestProcess_LastRankHighestQCInvalidRank() { + err := s.processor.Process(s.TimeoutLastRankFailedFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.PriorRankTimeoutCertificate.(*helper.TestTimeoutCertificate).LatestQuorumCert.(*helper.TestQuorumCertificate).Rank = t.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank + 1 // TC contains newer QC than Timeout State + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) +} + +// TestProcess_PreviousRankTimeoutCertificateRequiredButNotPresent tests that TimeoutProcessor fails with models.InvalidTimeoutError if +// timeout must contain a proof that sender legitimately entered timeout.Rank but doesn't have it. +func (s *TimeoutProcessorTestSuite) TestProcess_PreviousRankTimeoutCertificateRequiredButNotPresent() { + // if last rank is not successful(timeout.Rank != timeout.HighestQC.Rank+1) then this + // timeout must contain valid timeout.PriorRankTimeoutCertificate + err := s.processor.Process(s.TimeoutLastRankFailedFixture(func(t *models.TimeoutState[*helper.TestVote]) { + t.PriorRankTimeoutCertificate = nil + })) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) +} + +// TestProcess_IncludedQCInvalid tests that TimeoutProcessor correctly handles validation errors if +// timeout is well-formed but included QC is invalid +func (s *TimeoutProcessorTestSuite) TestProcess_IncludedQCInvalid() { + timeout := s.TimeoutLastRankSuccessfulFixture() + + s.Run("invalid-qc-sentinel", func() { + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(models.InvalidQuorumCertificateError{}).Once() + + err := s.processor.Process(timeout) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.True(s.T(), models.IsInvalidQuorumCertificateError(err)) + }) + s.Run("invalid-qc-exception", func() { + exception := errors.New("validate-qc-failed") + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(exception).Once() + + err := s.processor.Process(timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + }) + s.Run("invalid-qc-err-rank-for-unknown-epoch", func() { + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(models.ErrRankUnknown).Once() + + err := s.processor.Process(timeout) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.NotErrorIs(s.T(), err, models.ErrRankUnknown) + }) +} + +// TestProcess_IncludedTCInvalid tests that TimeoutProcessor correctly handles validation errors if +// timeout is well-formed but included TC is invalid +func (s *TimeoutProcessorTestSuite) TestProcess_IncludedTCInvalid() { + timeout := s.TimeoutLastRankFailedFixture() + + s.Run("invalid-tc-sentinel", func() { + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil) + s.validator.On("ValidateTimeoutCertificate", timeout.PriorRankTimeoutCertificate).Return(models.InvalidTimeoutCertificateError{}) + + err := s.processor.Process(timeout) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.True(s.T(), models.IsInvalidTimeoutCertificateError(err)) + }) + s.Run("invalid-tc-exception", func() { + exception := errors.New("validate-tc-failed") + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil) + s.validator.On("ValidateTimeoutCertificate", timeout.PriorRankTimeoutCertificate).Return(exception).Once() + + err := s.processor.Process(timeout) + require.ErrorIs(s.T(), err, exception) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + }) + s.Run("invalid-tc-err-rank-for-unknown-epoch", func() { + *s.validator = *mocks.NewValidator[*helper.TestState, *helper.TestVote](s.T()) + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil) + s.validator.On("ValidateTimeoutCertificate", timeout.PriorRankTimeoutCertificate).Return(models.ErrRankUnknown).Once() + + err := s.processor.Process(timeout) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.NotErrorIs(s.T(), err, models.ErrRankUnknown) + }) +} + +// TestProcess_ValidTimeout tests that processing a valid timeout succeeds without error +func (s *TimeoutProcessorTestSuite) TestProcess_ValidTimeout() { + s.Run("happy-path", func() { + timeout := s.TimeoutLastRankSuccessfulFixture() + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil).Once() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + s.sigAggregator.AssertCalled(s.T(), "VerifyAndAdd", (*timeout.Vote).ID, (*timeout.Vote).Signature, timeout.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank) + }) + s.Run("recovery-path", func() { + timeout := s.TimeoutLastRankFailedFixture() + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil).Once() + s.validator.On("ValidateTimeoutCertificate", timeout.PriorRankTimeoutCertificate).Return(nil).Once() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + s.sigAggregator.AssertCalled(s.T(), "VerifyAndAdd", (*timeout.Vote).ID, (*timeout.Vote).Signature, timeout.LatestQuorumCertificate.(*helper.TestQuorumCertificate).Rank) + }) +} + +// TestProcess_VerifyAndAddFailed tests different scenarios when TimeoutSignatureAggregator fails with error. +// We check all sentinel errors and exceptions in this scenario. +func (s *TimeoutProcessorTestSuite) TestProcess_VerifyAndAddFailed() { + timeout := s.TimeoutLastRankSuccessfulFixture() + s.validator.On("ValidateQuorumCertificate", timeout.LatestQuorumCertificate).Return(nil) + s.Run("invalid-signer", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), models.NewInvalidSignerError(fmt.Errorf(""))).Once() + err := s.processor.Process(timeout) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.True(s.T(), models.IsInvalidSignerError(err)) + }) + s.Run("invalid-signature", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), models.ErrInvalidSignature).Once() + err := s.processor.Process(timeout) + require.True(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.ErrorIs(s.T(), err, models.ErrInvalidSignature) + }) + s.Run("duplicated-signer", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), models.NewDuplicatedSignerErrorf("")).Once() + err := s.processor.Process(timeout) + require.True(s.T(), models.IsDuplicatedSignerError(err)) + // this shouldn't be wrapped in invalid timeout + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + }) + s.Run("verify-exception", func() { + *s.sigAggregator = *mocks.NewTimeoutSignatureAggregator(s.T()) + exception := errors.New("verify-exception") + s.sigAggregator.On("VerifyAndAdd", mock.Anything, mock.Anything, mock.Anything). + Return(uint64(0), exception).Once() + err := s.processor.Process(timeout) + require.False(s.T(), models.IsInvalidTimeoutError[*helper.TestVote](err)) + require.ErrorIs(s.T(), err, exception) + }) +} + +// TestProcess_CreatingTC is a test for happy path single threaded signature aggregation and TC creation +// Each replica commits unique timeout state, this object gets processed by TimeoutProcessor. After collecting +// enough weight we expect a TC to be created. All further operations should be no-op, only one TC should be created. +func (s *TimeoutProcessorTestSuite) TestProcess_CreatingTC() { + // consider next situation: + // last successful rank was N, after this we weren't able to get a proposal with QC for + // len(participants) ranks, but in each rank QC was created(but not distributed). + // In rank N+len(participants) each replica contributes with unique highest QC. + lastSuccessfulQC := helper.MakeQC(helper.WithQCRank(s.rank - uint64(len(s.participants)))) + previousRankTimeoutCert := helper.MakeTC(helper.WithTCRank(s.rank-1), + helper.WithTCNewestQC(lastSuccessfulQC)) + + var highQCRanks []uint64 + var timeouts []*models.TimeoutState[*helper.TestVote] + signers := s.participants[1:] + for i, signer := range signers { + qc := helper.MakeQC(helper.WithQCRank(lastSuccessfulQC.GetRank() + uint64(i+1))) + highQCRanks = append(highQCRanks, qc.GetRank()) + + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutNewestQC[*helper.TestVote](qc), + helper.WithTimeoutVote(&helper.TestVote{ID: signer.Identity(), Rank: s.rank}), + helper.WithTimeoutPreviousRankTimeoutCertificate[*helper.TestVote](previousRankTimeoutCert), + ) + timeouts = append(timeouts, timeout) + } + + // change tracker to require all except one signer to create TC + s.processor.tcTracker.minRequiredWeight = s.sigWeight * uint64(len(highQCRanks)) + + expectedSigBytes := make([]byte, 74) + expectedSig := &helper.TestAggregatedSignature{ + Signature: expectedSigBytes, + Bitmask: []byte{0b11111111, 0b00000111}, + PublicKey: make([]byte, 585), + } + s.validator.On("ValidateQuorumCertificate", mock.Anything).Return(nil) + s.validator.On("ValidateTimeoutCertificate", mock.Anything).Return(nil) + s.notifier.On("OnPartialTimeoutCertificateCreated", s.rank, mock.Anything, previousRankTimeoutCert).Return(nil).Once() + s.notifier.On("OnTimeoutCertificateConstructedFromTimeouts", mock.Anything).Run(func(args mock.Arguments) { + newestQC := timeouts[len(timeouts)-1].LatestQuorumCertificate + tc := args.Get(0).(models.TimeoutCertificate) + // ensure that TC contains correct fields + expectedTC := &helper.TestTimeoutCertificate{ + Rank: s.rank, + LatestRanks: highQCRanks, + LatestQuorumCert: newestQC, + AggregatedSignature: expectedSig, + } + require.Equal(s.T(), expectedTC, tc) + }).Return(nil).Once() + s.voting.On("FinalizeTimeout", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&helper.TestTimeoutCertificate{ + Filter: nil, + Rank: s.rank, + LatestRanks: highQCRanks, + LatestQuorumCert: timeouts[len(timeouts)-1].LatestQuorumCertificate, + AggregatedSignature: &helper.TestAggregatedSignature{ + PublicKey: make([]byte, 585), + Signature: make([]byte, 74), + Bitmask: []byte{0b11111111, 0b00000111}, + }, + }, nil) + + signersData := make([]consensus.TimeoutSignerInfo, 0) + for i, signer := range signers { + signersData = append(signersData, consensus.TimeoutSignerInfo{ + NewestQCRank: highQCRanks[i], + Signer: signer.Identity(), + }) + } + s.sigAggregator.On("Aggregate").Return(signersData, expectedSig, nil) + + for _, timeout := range timeouts { + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + } + s.notifier.AssertExpectations(s.T()) + s.sigAggregator.AssertExpectations(s.T()) + + // add extra timeout, make sure we don't create another TC + // should be no-op + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutNewestQC[*helper.TestVote](helper.MakeQC(helper.WithQCRank(lastSuccessfulQC.GetRank()))), + helper.WithTimeoutVote(&helper.TestVote{ + ID: s.participants[0].Identity(), + Rank: s.rank, + }), + helper.WithTimeoutPreviousRankTimeoutCertificate[*helper.TestVote](nil), + ) + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + + s.notifier.AssertExpectations(s.T()) + s.validator.AssertExpectations(s.T()) +} + +// TestProcess_ConcurrentCreatingTC tests a scenario where multiple goroutines process timeout at same time, +// we expect only one TC created in this scenario. +func (s *TimeoutProcessorTestSuite) TestProcess_ConcurrentCreatingTC() { + s.validator.On("ValidateQuorumCertificate", mock.Anything).Return(nil) + s.notifier.On("OnPartialTimeoutCertificateCreated", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + s.notifier.On("OnTimeoutCertificateConstructedFromTimeouts", mock.Anything).Return(nil).Once() + + signersData := make([]consensus.TimeoutSignerInfo, 0, len(s.participants)) + for _, signer := range s.participants { + signersData = append(signersData, consensus.TimeoutSignerInfo{ + NewestQCRank: 0, + Signer: signer.Identity(), + }) + } + // don't care about actual data + s.sigAggregator.On("Aggregate").Return(signersData, &helper.TestAggregatedSignature{PublicKey: make([]byte, 585), Signature: make([]byte, 74), Bitmask: []byte{0b11111111, 0b00000111}}, nil) + var startupWg, shutdownWg sync.WaitGroup + + newestQC := helper.MakeQC(helper.WithQCRank(s.rank - 1)) + s.voting.On("FinalizeTimeout", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&helper.TestTimeoutCertificate{ + Filter: nil, + Rank: s.rank, + LatestRanks: []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + LatestQuorumCert: newestQC, + AggregatedSignature: &helper.TestAggregatedSignature{ + PublicKey: make([]byte, 585), + Signature: make([]byte, 74), + Bitmask: []byte{0b11111111, 0b00000111}, + }, + }, nil) + + startupWg.Add(1) + // prepare goroutines, so they are ready to submit a timeout at roughly same time + for i, signer := range s.participants { + shutdownWg.Add(1) + timeout := helper.TimeoutStateFixture( + helper.WithTimeoutStateRank[*helper.TestVote](s.rank), + helper.WithTimeoutNewestQC[*helper.TestVote](newestQC), + helper.WithTimeoutVote(&helper.TestVote{ + ID: signer.Identity(), + Rank: s.rank, + }), + helper.WithTimeoutPreviousRankTimeoutCertificate[*helper.TestVote](nil), + ) + go func(i int, timeout *models.TimeoutState[*helper.TestVote]) { + defer shutdownWg.Done() + startupWg.Wait() + err := s.processor.Process(timeout) + require.NoError(s.T(), err) + }(i, timeout) + } + + startupWg.Done() + + // wait for all routines to finish + shutdownWg.Wait() +} + +// TestTimeoutProcessor_BuildVerifyTC tests a complete path from creating timeouts to collecting timeouts and then +// building & verifying TC. +// This test emulates the most complex scenario where TC consists of TimeoutStates that are structurally different. +// Let's consider a case where at some rank N consensus committee generated both QC and TC, resulting in nodes differently entering rank N+1. +// When constructing TC for rank N+1 some replicas will contribute with TO{Rank:N+1, NewestQC.Rank: N, PreviousRankTimeoutCertificate: nil} +// while others with TO{Rank:N+1, NewestQC.Rank: N-1, PreviousRankTimeoutCertificate: TC{Rank: N, NewestQC.Rank: N-1}}. +// This results in multi-message BLS signature with messages picked from set M={N-1,N}. +// We have to be able to construct a valid TC for rank N+1 and successfully validate it. +// We start by building a valid QC for rank N-1, that will be included in every TimeoutState at rank N. +// Right after we create a valid QC for rank N. We need to have valid QCs since TimeoutProcessor performs complete validation of TimeoutState. +// Then we create a valid cryptographically signed timeout for each signer. Created timeouts are feed to TimeoutProcessor +// which eventually creates a TC after seeing processing enough objects. After we verify if TC was correctly constructed +// and if it doesn't violate protocol rules. At this point we have QC for rank N-1, both QC and TC for rank N. +// After constructing valid objects we will repeat TC creation process and create a TC for rank N+1 where replicas contribute +// with structurally different TimeoutStates to make sure that TC is correctly built and can be successfully validated. +func TestTimeoutProcessor_BuildVerifyTC(t *testing.T) { + // signers hold objects that are created with private key and can sign votes and proposals + signers := make(map[models.Identity]*verification.Signer[*helper.TestState, *helper.TestVote, *helper.TestPeer]) + // prepare proving signers, each signer has its own private/public key pair + // identities must be in canonical order + provingSigners := helper.WithWeightedIdentityList(11) + leader := provingSigners[0] + rank := uint64(rand.Uint32() + 100) + + state := helper.MakeState(helper.WithStateRank[*helper.TestState](rank-1), + helper.WithStateProposer[*helper.TestState](leader.Identity())) + votingProviders := []*mocks.VotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer]{} + for _, s := range provingSigners { + v := mocks.NewVotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer](t) + votingProviders = append(votingProviders, v) + vote := &helper.TestVote{ + ID: s.Identity(), + Rank: rank - 1, + Signature: make([]byte, 74), + Timestamp: uint64(time.Now().UnixMilli()), + StateID: state.Identifier, + } + v.On("SignVote", mock.Anything, mock.Anything).Return(&vote, nil).Once() + signers[s.Identity()] = verification.NewSigner(v) + } + + // utility function which generates a valid timeout for every signer + createTimeouts := func(participants []models.WeightedIdentity, rank uint64, newestQC models.QuorumCertificate, previousRankTimeoutCert models.TimeoutCertificate) []*models.TimeoutState[*helper.TestVote] { + timeouts := make([]*models.TimeoutState[*helper.TestVote], 0, len(participants)) + for _, signer := range participants { + timeout, err := signers[signer.Identity()].CreateTimeout(rank, newestQC, previousRankTimeoutCert) + require.NoError(t, err) + timeouts = append(timeouts, timeout) + } + return timeouts + } + + provingSignersSkeleton := provingSigners + + committee := mocks.NewDynamicCommittee(t) + committee.On("IdentitiesByRank", mock.Anything).Return(provingSignersSkeleton, nil) + committee.On("IdentitiesByState", mock.Anything).Return(provingSigners, nil) + committee.On("QuorumThresholdForRank", mock.Anything).Return(uint64(8000), nil) + committee.On("TimeoutThresholdForRank", mock.Anything).Return(uint64(8000), nil) + + // create first QC for rank N-1, this will be our olderQC + olderQC := createRealQC(t, committee, provingSignersSkeleton, signers, state) + // now create a second QC for rank N, this will be our newest QC + nextState := helper.MakeState( + helper.WithStateRank[*helper.TestState](rank), + helper.WithStateProposer[*helper.TestState](leader.Identity()), + helper.WithStateQC[*helper.TestState](olderQC)) + + for i, vp := range votingProviders { + vote := &helper.TestVote{ + ID: provingSigners[i].Identity(), + Rank: rank, + Signature: make([]byte, 74), + Timestamp: uint64(time.Now().UnixMilli()), + StateID: nextState.Identifier, + } + vp.On("SignVote", mock.Anything, mock.Anything).Return(&vote, nil).Once() + tvote := &helper.TestVote{ + ID: provingSigners[i].Identity(), + Rank: rank, + Signature: make([]byte, 74), + Timestamp: uint64(time.Now().UnixMilli()), + } + vp.On("SignTimeoutVote", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&tvote, nil) + } + newestQC := createRealQC(t, committee, provingSignersSkeleton, signers, nextState) + + // At this point we have created two QCs for round N-1 and N. + // Next step is create a TC for rank N. + + // create verifier that will do crypto checks of created TC + verifier := &mocks.Verifier[*helper.TestVote]{} + verifier.On("VerifyQuorumCertificate", mock.Anything).Return(nil) + verifier.On("VerifyTimeoutCertificate", mock.Anything).Return(nil) + + // create validator which will do compliance and crypto checks of created TC + validator := validator.NewValidator[*helper.TestState, *helper.TestVote](committee, verifier) + + var previousRankTimeoutCert models.TimeoutCertificate + onTCCreated := func(args mock.Arguments) { + tc := args.Get(0).(models.TimeoutCertificate) + // check if resulted TC is valid + err := validator.ValidateTimeoutCertificate(tc) + require.NoError(t, err) + previousRankTimeoutCert = tc + } + + sigagg := mocks.NewSignatureAggregator(t) + sigagg.On("VerifySignatureRaw", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true) + sigagg.On("Aggregate", mock.Anything, mock.Anything).Return(&helper.TestAggregatedSignature{PublicKey: make([]byte, 585), Signature: make([]byte, 74), Bitmask: []byte{0b11111111, 0b00000111}}, nil) + + aggregator, err := NewTimeoutSignatureAggregator(sigagg, rank, provingSignersSkeleton, []byte{}) + require.NoError(t, err) + + notifier := mocks.NewTimeoutCollectorConsumer[*helper.TestVote](t) + notifier.On("OnPartialTimeoutCertificateCreated", rank, olderQC, nil).Return().Once() + notifier.On("OnTimeoutCertificateConstructedFromTimeouts", mock.Anything).Run(onTCCreated).Return().Once() + voting := mocks.NewVotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer](t) + voting.On("FinalizeTimeout", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&helper.TestTimeoutCertificate{ + Filter: nil, + Rank: rank, + LatestRanks: []uint64{rank - 1, rank - 1, rank - 1, rank - 1, rank - 1, rank - 1, rank - 1, rank - 1}, + LatestQuorumCert: olderQC, + AggregatedSignature: &helper.TestAggregatedSignature{PublicKey: make([]byte, 585), Signature: make([]byte, 74), Bitmask: []byte{0b11111111, 0b00000111}}, + }, nil) + processor, err := NewTimeoutProcessor[*helper.TestState, *helper.TestVote, *helper.TestPeer](helper.Logger(), committee, validator, aggregator, notifier, voting) + require.NoError(t, err) + + // last rank was successful, no previousRankTimeoutCert in this case + timeouts := createTimeouts(provingSignersSkeleton, rank, olderQC, nil) + for _, timeout := range timeouts { + err := processor.Process(timeout) + require.NoError(t, err) + } + + notifier.AssertExpectations(t) + + // at this point we have created QCs for rank N-1 and N additionally a TC for rank N, we can create TC for rank N+1 + // with timeout states containing both QC and TC for rank N + + aggregator, err = NewTimeoutSignatureAggregator(sigagg, rank+1, provingSignersSkeleton, []byte{}) + require.NoError(t, err) + + notifier = mocks.NewTimeoutCollectorConsumer[*helper.TestVote](t) + notifier.On("OnPartialTimeoutCertificateCreated", rank+1, newestQC, mock.Anything).Return() + notifier.On("OnTimeoutCertificateConstructedFromTimeouts", mock.Anything).Run(onTCCreated).Return().Once() + processor, err = NewTimeoutProcessor[*helper.TestState, *helper.TestVote, *helper.TestPeer](helper.Logger(), committee, validator, aggregator, notifier, voting) + require.NoError(t, err) + + // part of committee will use QC, another part TC, this will result in aggregated signature consisting + // of two types of messages with ranks N-1 and N representing the newest QC known to replicas. + timeoutsWithQC := createTimeouts(provingSignersSkeleton[:len(provingSignersSkeleton)/2], rank+1, newestQC, nil) + timeoutsWithTC := createTimeouts(provingSignersSkeleton[len(provingSignersSkeleton)/2:], rank+1, olderQC, previousRankTimeoutCert) + timeouts = append(timeoutsWithQC, timeoutsWithTC...) + for _, timeout := range timeouts { + err := processor.Process(timeout) + require.NoError(t, err) + } + + notifier.AssertExpectations(t) +} + +// createRealQC is a helper function which generates a properly signed QC with real signatures for given state. +func createRealQC( + t *testing.T, + committee consensus.DynamicCommittee, + signers []models.WeightedIdentity, + signerObjects map[models.Identity]*verification.Signer[*helper.TestState, *helper.TestVote, *helper.TestPeer], + state *models.State[*helper.TestState], +) models.QuorumCertificate { + leader := signers[0] + leaderVote, err := signerObjects[leader.Identity()].CreateVote(state) + require.NoError(t, err) + proposal := helper.MakeSignedProposal(helper.WithProposal[*helper.TestState, *helper.TestVote](helper.MakeProposal(helper.WithState(state))), helper.WithVote[*helper.TestState](leaderVote)) + + var createdQC *models.QuorumCertificate + onQCCreated := func(qc models.QuorumCertificate) { + createdQC = &qc + } + + voteProcessorFactory := votecollector.NewVoteProcessorFactory[*helper.TestState, *helper.TestVote, *helper.TestPeer](committee, onQCCreated) + sigagg := mocks.NewSignatureAggregator(t) + sigagg.On("VerifySignatureRaw", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true) + sigagg.On("Aggregate", mock.Anything, mock.Anything).Return(&helper.TestAggregatedSignature{PublicKey: make([]byte, 585), Signature: make([]byte, 74), Bitmask: []byte{0b11111111, 0b00000111}}, nil) + + votingProvider := mocks.NewVotingProvider[*helper.TestState, *helper.TestVote, *helper.TestPeer](t) + votingProvider.On("FinalizeQuorumCertificate", mock.Anything, mock.Anything, mock.Anything).Return(&helper.TestQuorumCertificate{ + Filter: nil, + Rank: state.Rank, + FrameNumber: state.Rank, + Selector: state.Identifier, + Timestamp: time.Now().UnixMilli(), + AggregatedSignature: &helper.TestAggregatedSignature{PublicKey: make([]byte, 585), Signature: make([]byte, 74), Bitmask: []byte{0b11111111, 0b00000111}}, + }, nil) + voteProcessor, err := voteProcessorFactory.Create(helper.Logger(), proposal, []byte{}, sigagg, votingProvider) + require.NoError(t, err) + + for _, signer := range signers[1:] { + vote, err := signerObjects[signer.Identity()].CreateVote(state) + require.NoError(t, err) + err = voteProcessor.Process(vote) + require.NoError(t, err) + } + + require.NotNil(t, createdQC, "vote processor must create a valid QC at this point") + return *createdQC +} diff --git a/consensus/tracker/tracker.go b/consensus/tracker/tracker.go index 1330b7d..f316021 100644 --- a/consensus/tracker/tracker.go +++ b/consensus/tracker/tracker.go @@ -128,48 +128,48 @@ func (t *NewestStateTracker[StateT]) NewestState() *models.State[StateT] { return (*models.State[StateT])(t.newestState.Load()) } -// NewestPartialTcTracker tracks the newest partial TC (by rank) in a +// NewestPartialTimeoutCertificateTracker tracks the newest partial TC (by rank) in a // concurrency safe way. -type NewestPartialTcTracker struct { - newestPartialTc *atomic.UnsafePointer +type NewestPartialTimeoutCertificateTracker struct { + newestPartialTimeoutCertificate *atomic.UnsafePointer } -func NewNewestPartialTcTracker() *NewestPartialTcTracker { - tracker := &NewestPartialTcTracker{ - newestPartialTc: atomic.NewUnsafePointer(unsafe.Pointer(nil)), +func NewNewestPartialTimeoutCertificateTracker() *NewestPartialTimeoutCertificateTracker { + tracker := &NewestPartialTimeoutCertificateTracker{ + newestPartialTimeoutCertificate: atomic.NewUnsafePointer(unsafe.Pointer(nil)), } return tracker } -// Track updates local state of newestPartialTc if the provided instance is +// Track updates local state of newestPartialTimeoutCertificate if the provided instance is // newer (by rank). Concurrency safe. -func (t *NewestPartialTcTracker) Track( - partialTc *consensus.PartialTimeoutCertificateCreated, +func (t *NewestPartialTimeoutCertificateTracker) Track( + partialTimeoutCertificate *consensus.PartialTimeoutCertificateCreated, ) bool { // To record the newest value that we have ever seen, we need to use loop // with CAS atomic operation to make sure that we always write the latest // value in case of shared access to updated value. for { // take a snapshot - newestPartialTc := t.NewestPartialTc() + newestPartialTimeoutCertificate := t.NewestPartialTimeoutCertificate() // verify that our partial TC is from a newer rank - if newestPartialTc != nil && newestPartialTc.Rank >= partialTc.Rank { + if newestPartialTimeoutCertificate != nil && newestPartialTimeoutCertificate.Rank >= partialTimeoutCertificate.Rank { return false } // attempt to install new value, repeat in case of shared update. - if t.newestPartialTc.CompareAndSwap( - unsafe.Pointer(newestPartialTc), - unsafe.Pointer(partialTc), + if t.newestPartialTimeoutCertificate.CompareAndSwap( + unsafe.Pointer(newestPartialTimeoutCertificate), + unsafe.Pointer(partialTimeoutCertificate), ) { return true } } } -// NewestPartialTc returns the newest partial TC (by rank) tracked. +// NewestPartialTimeoutCertificate returns the newest partial TC (by rank) tracked. // Concurrency safe. func ( - t *NewestPartialTcTracker, -) NewestPartialTc() *consensus.PartialTimeoutCertificateCreated { - return (*consensus.PartialTimeoutCertificateCreated)(t.newestPartialTc.Load()) + t *NewestPartialTimeoutCertificateTracker, +) NewestPartialTimeoutCertificate() *consensus.PartialTimeoutCertificateCreated { + return (*consensus.PartialTimeoutCertificateCreated)(t.newestPartialTimeoutCertificate.Load()) } diff --git a/consensus/validator/validator.go b/consensus/validator/validator.go index 565ec17..df2fff2 100644 --- a/consensus/validator/validator.go +++ b/consensus/validator/validator.go @@ -242,12 +242,7 @@ func (v *Validator[StateT, VoteT]) ValidateQuorumCertificate( err = v.verifier.VerifyQuorumCertificate(qc) if err != nil { // Considerations about other errors that `VerifyQC` could return: - // * models.InvalidSignerError: for the time being, we assume that _every_ - // HotStuff participant is also a member of the random beacon committee. - // Consequently, `InvalidSignerError` should not occur atm. - // TODO: if the random beacon committee is a strict subset of the - // HotStuff committee, we expect `models.InvalidSignerError` here - // during normal operations. + // * models.InvalidSignerError // * models.InsufficientSignaturesError: we previously checked the total // weight of all signers meets the supermajority threshold, which is a // _positive_ number. Hence, there must be at least one signer. Hence, @@ -470,12 +465,7 @@ func (v *Validator[StateT, VoteT]) ValidateVote(vote *VoteT) ( err = v.verifier.VerifyVote(vote) if err != nil { // Theoretically, `VerifyVote` could also return a - // `models.InvalidSignerError`. However, for the time being, we assume that - // _every_ HotStuff participant is also a member of the random beacon - // committee. Consequently, `InvalidSignerError` should not occur atm. - // TODO: if the random beacon committee is a strict subset of the HotStuff - // committee, we expect `models.InvalidSignerError` here during normal - // operations. + // `models.InvalidSignerError`. if models.IsInvalidFormatError(err) || errors.Is(err, models.ErrInvalidSignature) { return nil, newInvalidVoteError(vote, err) diff --git a/consensus/verification/common.go b/consensus/verification/common.go index 0566b32..cbdc58a 100644 --- a/consensus/verification/common.go +++ b/consensus/verification/common.go @@ -15,7 +15,7 @@ import ( // message without having the full state contents. func MakeVoteMessage(rank uint64, stateID models.Identity) []byte { msg := []byte{} - binary.BigEndian.PutUint64(msg, rank) + binary.BigEndian.AppendUint64(msg, rank) msg = append(msg, stateID[:]...) return msg } diff --git a/consensus/vote_aggregator.go b/consensus/vote_aggregator.go index 60bfe72..386ad03 100644 --- a/consensus/vote_aggregator.go +++ b/consensus/vote_aggregator.go @@ -24,7 +24,7 @@ type VoteAggregator[StateT models.Unique, VoteT models.Unique] interface { // `VoteAggregator` and processed _asynchronously_ by the VoteAggregator's // internal worker routines. // CAUTION: we expect that the input state's validity has been confirmed prior - // to calling AddState, including the proposer's signature. Otherwise, + // to calling AddState, including the proposer's consensus. Otherwise, // VoteAggregator might crash or exhibit undefined behaviour. AddState(state *models.SignedProposal[StateT, VoteT]) diff --git a/consensus/vote_collector.go b/consensus/vote_collector.go index ebeceea..40084f5 100644 --- a/consensus/vote_collector.go +++ b/consensus/vote_collector.go @@ -123,12 +123,22 @@ type VerifyingVoteProcessor[ // VoteProcessorFactory is a factory that can be used to create a verifying vote // processors for a specific proposal. Depending on factory implementation it // will return processors for consensus or collection clusters -type VoteProcessorFactory[StateT models.Unique, VoteT models.Unique] interface { +type VoteProcessorFactory[ + StateT models.Unique, + VoteT models.Unique, + PeerIDT models.Unique, +] interface { // Create instantiates a VerifyingVoteProcessor for processing votes for a // specific proposal. Caller can be sure that proposal vote was successfully // verified and processed. Expected error returns during normal operations: // * models.InvalidProposalError - proposal has invalid proposer vote - Create(tracer TraceLogger, proposal *models.SignedProposal[StateT, VoteT]) ( + Create( + tracer TraceLogger, + proposal *models.SignedProposal[StateT, VoteT], + dsTag []byte, + aggregator SignatureAggregator, + votingProvider VotingProvider[StateT, VoteT, PeerIDT], + ) ( VerifyingVoteProcessor[StateT, VoteT], error, ) diff --git a/consensus/voteaggregator/vote_aggregator.go b/consensus/voteaggregator/vote_aggregator.go index 733bb4e..679ecb3 100644 --- a/consensus/voteaggregator/vote_aggregator.go +++ b/consensus/voteaggregator/vote_aggregator.go @@ -224,7 +224,7 @@ func (va *VoteAggregator[StateT, VoteT]) processQueuedVote(vote *VoteT) error { // processQueuedState performs actual processing of queued state proposals, this // method is called from multiple concurrent goroutines. // CAUTION: we expect that the input state's validity has been confirmed prior -// to calling AddState, including the proposer's signature. Otherwise, +// to calling AddState, including the proposer's consensus. Otherwise, // VoteAggregator might crash or exhibit undefined behaviour. No errors are // expected during normal operation. func (va *VoteAggregator[StateT, VoteT]) processQueuedState( @@ -302,7 +302,7 @@ func (va *VoteAggregator[StateT, VoteT]) AddVote(vote *VoteT) { // `VoteAggregator` and processed _asynchronously_ by the VoteAggregator's // internal worker routines. // CAUTION: we expect that the input state's validity has been confirmed prior -// to calling AddState, including the proposer's signature. Otherwise, +// to calling AddState, including the proposer's consensus. Otherwise, // VoteAggregator might crash or exhibit undefined behaviour. func (va *VoteAggregator[StateT, VoteT]) AddState( state *models.SignedProposal[StateT, VoteT], diff --git a/consensus/voteaggregator/vote_collectors.go b/consensus/voteaggregator/vote_collectors.go index 9644e14..a21b34b 100644 --- a/consensus/voteaggregator/vote_collectors.go +++ b/consensus/voteaggregator/vote_collectors.go @@ -32,17 +32,11 @@ type VoteCollectors[StateT models.Unique, VoteT models.Unique] struct { var _ consensus.VoteCollectors[*nilUnique, *nilUnique] = (*VoteCollectors[*nilUnique, *nilUnique])(nil) func NewVoteCollectors[StateT models.Unique, VoteT models.Unique]( - ctx context.Context, tracer consensus.TraceLogger, lowestRetainedRank uint64, workerPool consensus.Workerpool, factoryMethod NewCollectorFactoryMethod[StateT, VoteT], ) *VoteCollectors[StateT, VoteT] { - go func() { - <-ctx.Done() // wait for parent context to signal shutdown - workerPool.StopWait() // wait till all workers exit - }() - return &VoteCollectors[StateT, VoteT]{ tracer: tracer, lowestRetainedRank: lowestRetainedRank, @@ -52,6 +46,14 @@ func NewVoteCollectors[StateT models.Unique, VoteT models.Unique]( } } +func (v *VoteCollectors[StateT, VoteT]) Start(ctx context.Context) error { + go func() { + <-ctx.Done() // wait for parent context to signal shutdown + v.workerPool.StopWait() // wait till all workers exit + }() + return nil +} + // GetOrCreateCollector retrieves the consensus.VoteCollector for the specified // rank or creates one if none exists. // - (collector, true, nil) if no collector can be found by the rank, and a diff --git a/consensus/votecollector/common.go b/consensus/votecollector/common.go index e58545b..2ba6a83 100644 --- a/consensus/votecollector/common.go +++ b/consensus/votecollector/common.go @@ -44,21 +44,21 @@ func (c *NoopProcessor[VoteT]) Status() consensus.VoteCollectorStatus { // state but for a different stateID func EnsureVoteForState[StateT models.Unique, VoteT models.Unique]( vote *VoteT, - state *StateT, + state *models.State[StateT], ) error { - if (*vote).GetRank() != (*state).GetRank() { + if (*vote).GetRank() != state.Rank { return fmt.Errorf( "vote %v has rank %d while state's rank is %d: %w ", (*vote).Identity(), (*vote).GetRank(), - (*state).GetRank(), + state.Rank, VoteForIncompatibleRankError, ) } - if (*vote).Source() != (*state).Identity() { + if (*vote).Source() != state.Identifier { return fmt.Errorf( "expecting only votes for state %v, but vote %v is for state %v: %w ", - (*state).Identity(), + state.Identifier, (*vote).Identity(), (*vote).Source(), VoteForIncompatibleStateError, diff --git a/consensus/votecollector/factory.go b/consensus/votecollector/factory.go index 95ccc1d..9a9d337 100644 --- a/consensus/votecollector/factory.go +++ b/consensus/votecollector/factory.go @@ -17,13 +17,17 @@ import ( // `consensus.VoteProcessorFactory` by itself. The VoteProcessorFactory adds the // missing logic to verify the proposer's vote, by wrapping the baseFactory // (decorator pattern). -type baseFactory[StateT models.Unique, VoteT models.Unique] func( +type baseFactory[ + StateT models.Unique, + VoteT models.Unique, + PeerIDT models.Unique, +] func( tracer consensus.TraceLogger, state *models.State[StateT], -) ( - consensus.VerifyingVoteProcessor[StateT, VoteT], - error, -) + dsTag []byte, + aggregator consensus.SignatureAggregator, + votingProvider consensus.VotingProvider[StateT, VoteT, PeerIDT], +) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) // VoteProcessorFactory implements `consensus.VoteProcessorFactory`. Its main // purpose is to construct instances of VerifyingVoteProcessors for a given @@ -34,21 +38,34 @@ type baseFactory[StateT models.Unique, VoteT models.Unique] func( // Thereby, VoteProcessorFactory guarantees that only proposals with valid // proposer vote are accepted (as per API specification). Otherwise, an // `models.InvalidProposalError` is returned. -type VoteProcessorFactory[StateT models.Unique, VoteT models.Unique] struct { - baseFactory baseFactory[StateT, VoteT] +type VoteProcessorFactory[ + StateT models.Unique, + VoteT models.Unique, + PeerIDT models.Unique, +] struct { + baseFactory baseFactory[StateT, VoteT, PeerIDT] } -var _ consensus.VoteProcessorFactory[*nilUnique, *nilUnique] = (*VoteProcessorFactory[*nilUnique, *nilUnique])(nil) +var _ consensus.VoteProcessorFactory[*nilUnique, *nilUnique, *nilUnique] = (*VoteProcessorFactory[*nilUnique, *nilUnique, *nilUnique])(nil) // Create instantiates a VerifyingVoteProcessor for the given state proposal. // A VerifyingVoteProcessor are only created for proposals with valid proposer // votes. Expected error returns during normal operations: // * models.InvalidProposalError - proposal has invalid proposer vote -func (f *VoteProcessorFactory[StateT, VoteT]) Create( +func (f *VoteProcessorFactory[StateT, VoteT, PeerIDT]) Create( tracer consensus.TraceLogger, proposal *models.SignedProposal[StateT, VoteT], + dsTag []byte, + aggregator consensus.SignatureAggregator, + votingProvider consensus.VotingProvider[StateT, VoteT, PeerIDT], ) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) { - processor, err := f.baseFactory(tracer, proposal.State) + processor, err := f.baseFactory( + tracer, + proposal.State, + dsTag, + aggregator, + votingProvider, + ) if err != nil { return nil, fmt.Errorf( "instantiating vote processor for state %v failed: %w", @@ -88,12 +105,12 @@ func NewVoteProcessorFactory[ ]( committee consensus.DynamicCommittee, onQCCreated consensus.OnQuorumCertificateCreated, -) *VoteProcessorFactory[StateT, VoteT] { +) *VoteProcessorFactory[StateT, VoteT, PeerIDT] { base := &provingVoteProcessorFactoryBase[StateT, VoteT, PeerIDT]{ committee: committee, onQCCreated: onQCCreated, } - return &VoteProcessorFactory[StateT, VoteT]{ + return &VoteProcessorFactory[StateT, VoteT, PeerIDT]{ baseFactory: base.Create, } } @@ -113,12 +130,15 @@ func NewBootstrapVoteProcessor[ committee consensus.DynamicCommittee, state *models.State[StateT], onQCCreated consensus.OnQuorumCertificateCreated, + dsTag []byte, + aggregator consensus.SignatureAggregator, + votingProvider consensus.VotingProvider[StateT, VoteT, PeerIDT], ) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) { factory := &provingVoteProcessorFactoryBase[StateT, VoteT, PeerIDT]{ committee: committee, onQCCreated: onQCCreated, } - return factory.Create(tracer, state) + return factory.Create(tracer, state, dsTag, aggregator, votingProvider) } // Type used to satisfy generic arguments in compiler time type assertion check diff --git a/consensus/votecollector/statemachine.go b/consensus/votecollector/statemachine.go index 5b96648..6ae2be2 100644 --- a/consensus/votecollector/statemachine.go +++ b/consensus/votecollector/statemachine.go @@ -5,9 +5,10 @@ import ( "fmt" "sync" - "github.com/rs/zerolog" "go.uber.org/atomic" + "source.quilibrium.com/quilibrium/monorepo/consensus" + "source.quilibrium.com/quilibrium/monorepo/consensus/models" "source.quilibrium.com/quilibrium/monorepo/consensus/voteaggregator" ) @@ -29,7 +30,7 @@ type VerifyingVoteProcessorFactory[ // states of vote collector type VoteCollector[StateT models.Unique, VoteT models.Unique] struct { sync.Mutex - log zerolog.Logger + tracer consensus.TraceLogger workers consensus.Workers notifier consensus.VoteAggregationConsumer[StateT, VoteT] createVerifyingProcessor VerifyingVoteProcessorFactory[StateT, VoteT] @@ -81,16 +82,16 @@ func NewStateMachine[StateT models.Unique, VoteT models.Unique]( verifyingVoteProcessorFactory VerifyingVoteProcessorFactory[StateT, VoteT], ) *VoteCollector[StateT, VoteT] { sm := &VoteCollector[StateT, VoteT]{ - tracer: tracer + tracer: tracer, workers: workers, notifier: notifier, createVerifyingProcessor: verifyingVoteProcessorFactory, - votesCache: *NewVotesCache[StateT, VoteT](rank), + votesCache: *NewVotesCache[VoteT](rank), } // without a state, we don't process votes (only cache them) - sm.votesProcessor.Store(&atomicValueWrapper{ - processor: NewNoopCollector(consensus.VoteCollectorStatusCaching), + sm.votesProcessor.Store(&atomicValueWrapper[VoteT]{ + processor: NewNoopCollector[VoteT](consensus.VoteCollectorStatusCaching), }) return sm } @@ -105,7 +106,7 @@ func (m *VoteCollector[StateT, VoteT]) AddVote(vote *VoteT) error { if errors.Is(err, RepeatedVoteErr) { return nil } - doubleVoteErr, isDoubleVoteErr := models.AsDoubleVoteError(err) + doubleVoteErr, isDoubleVoteErr := models.AsDoubleVoteError[VoteT](err) if isDoubleVoteErr { m.notifier.OnDoubleVotingDetected( doubleVoteErr.FirstVote, @@ -115,8 +116,8 @@ func (m *VoteCollector[StateT, VoteT]) AddVote(vote *VoteT) error { } return fmt.Errorf( "internal error adding vote %v to cache for state %v: %w", - vote.ID(), - vote.Identifier, + (*vote).Identity(), + (*vote).Source(), err, ) } @@ -143,8 +144,8 @@ func (m *VoteCollector[StateT, VoteT]) AddVote(vote *VoteT) error { } return fmt.Errorf( "internal error processing vote %v for state %v: %w", - vote.ID(), - vote.Identifier, + (*vote).Identity(), + (*vote).Source(), err, ) } @@ -159,7 +160,7 @@ func (m *VoteCollector[StateT, VoteT]) processVote(vote *VoteT) error { currentState := processor.Status() err := processor.Process(vote) if err != nil { - if invalidVoteErr, ok := models.AsInvalidVoteError(err); ok { + if invalidVoteErr, ok := models.AsInvalidVoteError[VoteT](err); ok { m.notifier.OnInvalidVoteDetected(*invalidVoteErr) return nil } @@ -168,7 +169,7 @@ func (m *VoteCollector[StateT, VoteT]) processVote(vote *VoteT) error { // double voting. This scenario is possible if leader submits their vote // additionally to the vote in proposal. if models.IsDuplicatedSignerError(err) { - m.tracer.Trace(fmt.Sprintf("duplicated signer %x", vote.SignerID)) + m.tracer.Trace(fmt.Sprintf("duplicated signer %x", (*vote).Identity())) return nil } return err @@ -207,7 +208,9 @@ func (m *VoteCollector[StateT, VoteT]) Rank() uint64 { // CachingVotes -> VerifyingVotes // CachingVotes -> Invalid // VerifyingVotes -> Invalid -func (m *VoteCollector[StateT, VoteT]) ProcessState(proposal *models.SignedProposal) error { +func (m *VoteCollector[StateT, VoteT]) ProcessState( + proposal *models.SignedProposal[StateT, VoteT], +) error { if proposal.State.Rank != m.Rank() { return fmt.Errorf( @@ -240,9 +243,7 @@ func (m *VoteCollector[StateT, VoteT]) ProcessState(proposal *models.SignedPropo ) } - m.log.Info(). - Hex("state_id", proposal.State.Identifier[:]). - Msg("vote collector status changed from caching to verifying") + m.tracer.Trace("vote collector status changed from caching to verifying") m.processCachedVotes(proposal.State) @@ -251,7 +252,7 @@ func (m *VoteCollector[StateT, VoteT]) ProcessState(proposal *models.SignedPropo // Note: proposal equivocation is handled by consensus.Forks, so we don't // have to do anything else here. case consensus.VoteCollectorStatusVerifying: - verifyingProc, ok := proc.(consensus.VerifyingVoteProcessor) + verifyingProc, ok := proc.(consensus.VerifyingVoteProcessor[StateT, VoteT]) if !ok { return fmt.Errorf( "while processing state %v, found that VoteProcessor reports status %s but has an incompatible implementation type %T", @@ -296,25 +297,33 @@ func (m *VoteCollector[StateT, VoteT]) RegisterVoteConsumer( // `VoteCollectorStatusCaching` and replaces it by a newly-created // VerifyingVoteProcessor. // Error returns: -// * ErrDifferentCollectorState if the VoteCollector's state is _not_ -// `CachingVotes` -// * all other errors are unexpected and potential symptoms of internal bugs or -// state corruption (fatal) +// - ErrDifferentCollectorState if the VoteCollector's state is _not_ +// `CachingVotes` +// - all other errors are unexpected and potential symptoms of internal bugs +// or state corruption (fatal) func (m *VoteCollector[StateT, VoteT]) caching2Verifying( proposal *models.SignedProposal[StateT, VoteT], ) error { stateID := proposal.State.Identifier - newProc, err := m.createVerifyingProcessor(m.log, proposal) + newProc, err := m.createVerifyingProcessor(m.tracer, proposal) if err != nil { - return fmt.Errorf("failed to create VerifyingVoteProcessor for state %v: %w", stateID, err) + return fmt.Errorf( + "failed to create VerifyingVoteProcessor for state %v: %w", + stateID, + err, + ) } - newProcWrapper := &atomicValueWrapper{processor: newProc} + newProcWrapper := &atomicValueWrapper[VoteT]{processor: newProc} m.Lock() defer m.Unlock() proc := m.atomicLoadProcessor() if proc.Status() != consensus.VoteCollectorStatusCaching { - return fmt.Errorf("processors's current state is %s: %w", proc.Status().String(), ErrDifferentCollectorState) + return fmt.Errorf( + "processors's current state is %s: %w", + proc.Status().String(), + ErrDifferentCollectorState, + ) } m.votesProcessor.Store(newProcWrapper) return nil @@ -324,8 +333,8 @@ func (m *VoteCollector[StateT, VoteT]) terminateVoteProcessing() { if m.Status() == consensus.VoteCollectorStatusInvalid { return } - newProcWrapper := &atomicValueWrapper{ - processor: NewNoopCollector(consensus.VoteCollectorStatusInvalid), + newProcWrapper := &atomicValueWrapper[VoteT]{ + processor: NewNoopCollector[VoteT](consensus.VoteCollectorStatusInvalid), } m.Lock() @@ -334,11 +343,13 @@ func (m *VoteCollector[StateT, VoteT]) terminateVoteProcessing() { } // processCachedVotes feeds all cached votes into the VoteProcessor -func (m *VoteCollector[StateT, VoteT]) processCachedVotes(state *models.State) { +func (m *VoteCollector[StateT, VoteT]) processCachedVotes( + state *models.State[StateT], +) { cachedVotes := m.votesCache.All() - m.log.Info().Msgf("processing %d cached votes", len(cachedVotes)) + m.tracer.Trace(fmt.Sprintf("processing %d cached votes", len(cachedVotes))) for _, vote := range cachedVotes { - if vote.Identifier != state.Identifier { + if (*vote).Source() != state.Identifier { continue } @@ -346,7 +357,7 @@ func (m *VoteCollector[StateT, VoteT]) processCachedVotes(state *models.State) { voteProcessingTask := func() { err := m.processVote(stateVote) if err != nil { - m.log.Fatal().Err(err).Msg("internal error processing cached vote") + m.tracer.Error("internal error processing cached vote", err) } } m.workers.Submit(voteProcessingTask) diff --git a/consensus/votecollector/vote_cache.go b/consensus/votecollector/vote_cache.go index 6ef0dae..5963e1d 100644 --- a/consensus/votecollector/vote_cache.go +++ b/consensus/votecollector/vote_cache.go @@ -17,8 +17,8 @@ var ( // voteContainer container stores the vote and in index representing // the order in which the votes were received -type voteContainer struct { - *models.Vote +type voteContainer[VoteT models.Unique] struct { + Vote *VoteT index int } @@ -33,7 +33,7 @@ type voteContainer struct { type VotesCache[VoteT models.Unique] struct { lock sync.RWMutex rank uint64 - votes map[models.Identity]voteContainer // signerID -> first vote + votes map[models.Identity]voteContainer[VoteT] // signerID -> first vote voteConsumers []consensus.VoteConsumer[VoteT] } @@ -41,7 +41,7 @@ type VotesCache[VoteT models.Unique] struct { func NewVotesCache[VoteT models.Unique](rank uint64) *VotesCache[VoteT] { return &VotesCache[VoteT]{ rank: rank, - votes: make(map[models.Identity]voteContainer), + votes: make(map[models.Identity]voteContainer[VoteT]), } } @@ -85,7 +85,7 @@ func (vc *VotesCache[VoteT]) AddVote(vote *VoteT) error { } // previously unknown vote: (1) store and (2) forward to consumers - vc.votes[(*vote).Identity()] = voteContainer{vote, len(vc.votes)} + vc.votes[(*vote).Identity()] = voteContainer[VoteT]{vote, len(vc.votes)} for _, consumer := range vc.voteConsumers { consumer(vote) } diff --git a/consensus/votecollector/vote_processor.go b/consensus/votecollector/vote_processor.go index da213c3..3a93e03 100644 --- a/consensus/votecollector/vote_processor.go +++ b/consensus/votecollector/vote_processor.go @@ -39,6 +39,7 @@ func (f *provingVoteProcessorFactoryBase[StateT, VoteT, PeerIDT]) Create( state *models.State[StateT], dsTag []byte, aggregator consensus.SignatureAggregator, + votingProvider consensus.VotingProvider[StateT, VoteT, PeerIDT], ) (consensus.VerifyingVoteProcessor[StateT, VoteT], error) { allParticipants, err := f.committee.IdentitiesByState(state.Identifier) if err != nil { @@ -81,6 +82,7 @@ func (f *provingVoteProcessorFactoryBase[StateT, VoteT, PeerIDT]) Create( tracer: tracer, state: state, provingSigAggtor: provingSigAggtor, + votingProvider: votingProvider, onQCCreated: f.onQCCreated, minRequiredWeight: minRequiredWeight, done: *atomic.NewBool(false), @@ -92,7 +94,7 @@ func (f *provingVoteProcessorFactoryBase[StateT, VoteT, PeerIDT]) Create( // VoteProcessor implements the consensus.VerifyingVoteProcessor interface. // It processes hotstuff votes from a collector cluster, where participants vote -// in favour of a state by proving their proving key signature. +// in favour of a state by proving their proving key consensus. // Concurrency safe. type VoteProcessor[ StateT models.Unique,