diff --git a/.dockerignore b/.dockerignore index ff2bb43..36d102c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,10 @@ **/.vscode github.env Taskfile.yaml +.git # Rust target vdf/generated +bls48581/generated + diff --git a/node/config/config.go b/node/config/config.go index 7afd8ee..18da56e 100644 --- a/node/config/config.go +++ b/node/config/config.go @@ -150,14 +150,14 @@ var Signatories = []string{ "92cd8ee5362f3ae274a75ab9471024dbc144bff441ed8af7d19750ac512ff51e40e7f7b01e4f96b6345dd58878565948c3eb52c53f250b5080", "001a4cbfce5d9aeb7e20665b0d236721b228a32f0baee62ffa77f45b82ecaf577e8a38b7ef91fcf7d2d2d2b504f085461398d30b24abb1d700", "65b835071731c6e785bb2d107c7d85d8a537d79c435c3f42bb2f87027f93f858d7b37c598cef267a5db46e345f7a6f81969b465686657d1e00", - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "b6df0ebab6ea20cc2eb718db5873c07bb50cf239a16bb6306bbe0f24280664f99f732c4049b8eda1226067e70ffb81958834d486942a122100", + "3e087771c36098cb2d371711fd882d309b4caebbd06ded3077a975231344f027ad31c7069e76ba5070451d8eb5abf29bfeb34fcdf9ba906480", "57be2861faf0fffcbfd122c85c77010dce8f213030905781b85b6f345d912c7b5ace17797d9810899dfb8d13e7c8369595740725ab3dd5bd00", "61628beef8f6964466fd078d6a2b90a397ab0777a14b9728227fd19f36752f9451b1a8d780740a0b9a8ce3df5f89ca7b9ff17de9274a270980", - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "9ab76d775487c85c8e5aa0c5b3f961772967899a14644651031ae5f98ac197bee3f8880492c4fdba268716fc4b7c38ffcac370b663ac10b600", "81d63a45f068629f568de812f18be5807bfe828a830097f09cf02330d6acd35e3607401df3fda08b03b68ea6e68afd506b23506b11e87a0f80", "6e2872f73c4868c4286bef7bfe2f5479a41c42f4e07505efa4883c7950c740252e0eea78eef10c584b19b1dcda01f7767d3135d07c33244100", - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0ca6f5a9d7f86c1111be5edf31e26979918aa4fa3daae6de1120e05c2a09bdb8d2feeb084286a3347e06ced25530358cbc74c204d2a1753a00", } var unlock *SignedGenesisUnlock diff --git a/node/config/version.go b/node/config/version.go index 7f51307..aedd669 100644 --- a/node/config/version.go +++ b/node/config/version.go @@ -6,7 +6,7 @@ import ( ) func GetMinimumVersionCutoff() time.Time { - return time.Date(2024, time.November, 24, 0, 0, 0, 0, time.UTC) + return time.Date(2025, time.January, 13, 0, 0, 0, 0, time.UTC) } // Gets the minimum patch version – This should only be set in a release series @@ -43,7 +43,7 @@ func FormatVersion(version []byte) string { } func GetPatchNumber() byte { - return 0x00 + return 0x02 } func GetRCNumber() byte { diff --git a/node/consensus/data/consensus_frames.go b/node/consensus/data/consensus_frames.go index 99c659b..873374c 100644 --- a/node/consensus/data/consensus_frames.go +++ b/node/consensus/data/consensus_frames.go @@ -10,9 +10,12 @@ import ( "source.quilibrium.com/quilibrium/monorepo/node/config" "source.quilibrium.com/quilibrium/monorepo/node/consensus/data/internal" "source.quilibrium.com/quilibrium/monorepo/node/internal/frametime" + "source.quilibrium.com/quilibrium/monorepo/node/tries" + "github.com/iden3/go-iden3-crypto/poseidon" "github.com/libp2p/go-libp2p/core/peer" "github.com/pkg/errors" + mt "github.com/txaty/go-merkletree" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -27,15 +30,13 @@ func (e *DataClockConsensusEngine) syncWithMesh() error { if err != nil { return errors.Wrap(err, "sync") } + var doneChs []<-chan struct{} for { candidates := e.GetAheadPeers(max(latest.FrameNumber, e.latestFrameReceived)) if len(candidates) == 0 { break } for _, candidate := range candidates { - if candidate.MaxFrame <= max(latest.FrameNumber, e.latestFrameReceived) { - continue - } head, err := e.dataTimeReel.Head() if err != nil { return errors.Wrap(err, "sync") @@ -43,13 +44,24 @@ func (e *DataClockConsensusEngine) syncWithMesh() error { if latest.FrameNumber < head.FrameNumber { latest = head } - latest, err = e.syncWithPeer(latest, candidate.MaxFrame, candidate.PeerID) + if candidate.MaxFrame <= max(latest.FrameNumber, e.latestFrameReceived) { + continue + } + latest, doneChs, err = e.syncWithPeer(latest, doneChs, candidate.MaxFrame, candidate.PeerID) if err != nil { e.logger.Debug("error syncing frame", zap.Error(err)) } } } + for _, doneCh := range doneChs { + select { + case <-e.ctx.Done(): + return e.ctx.Err() + case <-doneCh: + } + } + e.logger.Info( "returning leader frame", zap.Uint64("frame_number", latest.FrameNumber), @@ -312,13 +324,13 @@ func (e *DataClockConsensusEngine) GetAheadPeers(frameNumber uint64) []internal. } func (e *DataClockConsensusEngine) syncWithPeer( - currentLatest *protobufs.ClockFrame, + latest *protobufs.ClockFrame, + doneChs []<-chan struct{}, maxFrame uint64, peerId []byte, -) (*protobufs.ClockFrame, error) { +) (*protobufs.ClockFrame, []<-chan struct{}, error) { e.syncingStatus = SyncStatusSynchronizing defer func() { e.syncingStatus = SyncStatusNotSyncing }() - latest := currentLatest e.logger.Info( "polling peer for new frames", zap.String("peer_id", peer.ID(peerId).String()), @@ -350,7 +362,7 @@ func (e *DataClockConsensusEngine) syncWithPeer( zap.Error(err), ) cooperative = false - return latest, errors.Wrap(err, "sync") + return latest, doneChs, errors.Wrap(err, "sync") } defer func() { if err := cc.Close(); err != nil { @@ -378,12 +390,12 @@ func (e *DataClockConsensusEngine) syncWithPeer( zap.Error(err), ) cooperative = false - return latest, errors.Wrap(err, "sync") + return latest, doneChs, errors.Wrap(err, "sync") } if response == nil { e.logger.Debug("received no response from peer") - return latest, nil + return latest, doneChs, nil } if response.ClockFrame == nil || @@ -391,7 +403,7 @@ func (e *DataClockConsensusEngine) syncWithPeer( response.ClockFrame.Timestamp < latest.Timestamp { e.logger.Debug("received invalid response from peer") cooperative = false - return latest, nil + return latest, doneChs, nil } e.logger.Info( "received new leading frame", @@ -406,12 +418,159 @@ func (e *DataClockConsensusEngine) syncWithPeer( if err := e.frameProver.VerifyDataClockFrame( response.ClockFrame, ); err != nil { - return nil, errors.Wrap(err, "sync") + return latest, doneChs, errors.Wrap(err, "sync") } - e.dataTimeReel.Insert(e.ctx, response.ClockFrame, true) + doneCh, err := e.dataTimeReel.Insert(e.ctx, response.ClockFrame) + if err != nil { + return latest, doneChs, errors.Wrap(err, "sync") + } + doneChs = append(doneChs, doneCh) latest = response.ClockFrame if latest.FrameNumber >= maxFrame { - return latest, nil + return latest, doneChs, nil + } + } +} + +func (e *DataClockConsensusEngine) initiateProvers( + latestFrame *protobufs.ClockFrame, +) { + if latestFrame.Timestamp > time.Now().UnixMilli()-60000 { + if !e.IsInProverTrie(e.pubSub.GetPeerID()) { + e.logger.Info("announcing prover join") + for _, eng := range e.executionEngines { + eng.AnnounceProverJoin() + break + } + } else { + if e.previousFrameProven != nil && + e.previousFrameProven.FrameNumber == latestFrame.FrameNumber { + return + } + + h, err := poseidon.HashBytes(e.pubSub.GetPeerID()) + if err != nil { + panic(err) + } + peerProvingKeyAddress := h.FillBytes(make([]byte, 32)) + + ring := -1 + if tries := e.GetFrameProverTries(); len(tries) > 1 { + for i, tries := range tries[1:] { + i := i + if tries.Contains(peerProvingKeyAddress) { + ring = i + } + } + } + + e.clientReconnectTest++ + if e.clientReconnectTest >= 10 { + e.tryReconnectDataWorkerClients() + e.clientReconnectTest = 0 + } + + previousTreeRoot := []byte{} + if e.previousTree != nil { + previousTreeRoot = e.previousTree.Root + } + outputs := e.PerformTimeProof(latestFrame, previousTreeRoot, latestFrame.Difficulty, ring) + if outputs == nil || len(outputs) < 3 { + e.logger.Info("workers not yet available for proving") + return + } + modulo := len(outputs) + var proofTree *mt.MerkleTree + var output [][]byte + if latestFrame.FrameNumber >= application.PROOF_FRAME_COMBINE_CUTOFF { + proofTree, output, err = tries.PackOutputIntoMultiPayloadAndProof( + outputs, + modulo, + latestFrame, + e.previousTree, + ) + } else { + proofTree, output, err = tries.PackOutputIntoPayloadAndProof( + outputs, + modulo, + latestFrame, + e.previousTree, + ) + } + if err != nil { + e.logger.Error( + "could not successfully pack proof, reattempting", + zap.Error(err), + ) + return + } + e.previousFrameProven = latestFrame + e.previousTree = proofTree + + mint := &protobufs.MintCoinRequest{ + Proofs: output, + } + if err := mint.SignED448(e.pubSub.GetPublicKey(), e.pubSub.SignMessage); err != nil { + e.logger.Error("could not sign mint", zap.Error(err)) + return + } + if err := mint.Validate(); err != nil { + e.logger.Error("mint validation failed", zap.Error(err)) + return + } + + e.logger.Info( + "submitting data proof", + zap.Int("ring", ring), + zap.Int("active_workers", len(outputs)), + zap.Uint64("frame_number", latestFrame.FrameNumber), + zap.Duration("frame_age", frametime.Since(latestFrame)), + ) + + if err := e.publishMessage(e.txFilter, mint.TokenRequest()); err != nil { + e.logger.Error("could not publish mint", zap.Error(err)) + } + + if e.config.Engine.AutoMergeCoins { + _, addrs, _, err := e.coinStore.GetCoinsForOwner( + peerProvingKeyAddress, + ) + if err != nil { + e.logger.Error( + "received error while iterating coins", + zap.Error(err), + ) + return + } + + if len(addrs) > 25 { + refs := []*protobufs.CoinRef{} + for _, addr := range addrs { + refs = append(refs, &protobufs.CoinRef{ + Address: addr, + }) + } + + merge := &protobufs.MergeCoinRequest{ + Coins: refs, + } + if err := merge.SignED448( + e.pubSub.GetPublicKey(), + e.pubSub.SignMessage, + ); err != nil { + e.logger.Error("could not sign merge", zap.Error(err)) + return + } + if err := merge.Validate(); err != nil { + e.logger.Error("merge validation failed", zap.Error(err)) + return + } + + if err := e.publishMessage(e.txFilter, merge.TokenRequest()); err != nil { + e.logger.Warn("could not publish merge", zap.Error(err)) + } + } + } } } } diff --git a/node/consensus/data/data_clock_consensus_engine.go b/node/consensus/data/data_clock_consensus_engine.go index 69cea60..3216c4f 100644 --- a/node/consensus/data/data_clock_consensus_engine.go +++ b/node/consensus/data/data_clock_consensus_engine.go @@ -10,6 +10,7 @@ import ( "sync" "time" + lru "github.com/hashicorp/golang-lru/v2" "github.com/libp2p/go-libp2p/p2p/discovery/backoff" "github.com/multiformats/go-multiaddr" mn "github.com/multiformats/go-multiaddr/net" @@ -29,6 +30,7 @@ import ( qtime "source.quilibrium.com/quilibrium/monorepo/node/consensus/time" qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/execution" + "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" "source.quilibrium.com/quilibrium/monorepo/node/internal/cas" "source.quilibrium.com/quilibrium/monorepo/node/internal/frametime" qgrpc "source.quilibrium.com/quilibrium/monorepo/node/internal/grpc" @@ -124,6 +126,7 @@ type DataClockConsensusEngine struct { previousHead *protobufs.ClockFrame engineMx sync.Mutex dependencyMapMx sync.Mutex + recentlyProcessedFrames *lru.Cache[string, struct{}] stagedTransactions *protobufs.TokenRequests stagedTransactionsSet map[string]struct{} stagedTransactionsMx sync.Mutex @@ -140,6 +143,7 @@ type DataClockConsensusEngine struct { infoMessageProcessorCh chan *pb.Message report *protobufs.SelfTestReport clients []protobufs.DataIPCServiceClient + clientsMx sync.Mutex grpcRateLimiter *RateLimiter previousFrameProven *protobufs.ClockFrame previousTree *mt.MerkleTree @@ -233,6 +237,11 @@ func NewDataClockConsensusEngine( panic(err) } + cache, err := lru.New[string, struct{}](25) + if err != nil { + panic(err) + } + ctx, cancel := context.WithCancel(context.Background()) e := &DataClockConsensusEngine{ ctx: ctx, @@ -281,6 +290,7 @@ func NewDataClockConsensusEngine( requestSyncCh: make(chan struct{}, 1), validationFilter: map[string]struct{}{}, clockFrameFragmentBuffer: clockFrameFragmentBuffer, + recentlyProcessedFrames: cache, } logger.Info("constructing consensus engine") @@ -561,19 +571,7 @@ func (e *DataClockConsensusEngine) Start() <-chan error { e.wg.Add(1) go func() { defer e.wg.Done() - if len(e.config.Engine.DataWorkerMultiaddrs) != 0 { - e.clients, err = e.createParallelDataClientsFromList() - if err != nil { - panic(err) - } - } else { - e.clients, err = e.createParallelDataClientsFromBaseMultiaddr( - e.config.Engine.DataWorkerCount, - ) - if err != nil { - panic(err) - } - } + e.createParallelDataWorkerClients() }() return errChan @@ -581,6 +579,7 @@ func (e *DataClockConsensusEngine) Start() <-chan error { func (e *DataClockConsensusEngine) PerformTimeProof( frame *protobufs.ClockFrame, + previousTreeRoot []byte, difficulty uint32, ring int, ) []mt.DataBlock { @@ -613,6 +612,12 @@ func (e *DataClockConsensusEngine) PerformTimeProof( wg := sync.WaitGroup{} wg.Add(len(actives)) + challengeOutput := []byte{} + if frame.FrameNumber >= application.PROOF_FRAME_COMBINE_CUTOFF { + challengeOutput = append(append([]byte{}, frame.Output...), previousTreeRoot...) + } else { + challengeOutput = frame.Output + } for i, client := range actives { i := i @@ -625,7 +630,7 @@ func (e *DataClockConsensusEngine) PerformTimeProof( &protobufs.ChallengeProofRequest{ PeerId: e.pubSub.GetPeerID(), Core: uint32(i), - Output: frame.Output, + Output: challengeOutput, FrameNumber: frame.FrameNumber, Difficulty: frame.Difficulty, }, @@ -823,211 +828,155 @@ func (e *DataClockConsensusEngine) createCommunicationKeys() error { return nil } -func (e *DataClockConsensusEngine) createParallelDataClientsFromListAndIndex( - index uint32, +func (e *DataClockConsensusEngine) connectToClient( + index int, + useList bool, ) ( protobufs.DataIPCServiceClient, error, ) { - ma, err := multiaddr.NewMultiaddr(e.config.Engine.DataWorkerMultiaddrs[index]) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - _, addr, err := mn.DialArgs(ma) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - ctx, cancel := context.WithTimeout(e.ctx, 1*time.Second) - defer cancel() - conn, err := qgrpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials( - insecure.NewCredentials(), - ), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(10*1024*1024), - grpc.MaxCallRecvMsgSize(10*1024*1024), - ), - grpc.WithBlock(), - ) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - client := protobufs.NewDataIPCServiceClient(conn) - - e.logger.Info( - "connected to data worker process", - zap.Uint32("client", index), - ) - return client, nil -} - -func ( - e *DataClockConsensusEngine, -) createParallelDataClientsFromBaseMultiaddrAndIndex( - index uint32, -) ( - protobufs.DataIPCServiceClient, - error, -) { - e.logger.Info( - "re-connecting to data worker process", - zap.Uint32("client", index), - ) - - ma, err := multiaddr.NewMultiaddr( - fmt.Sprintf( - e.config.Engine.DataWorkerBaseListenMultiaddr, - int(e.config.Engine.DataWorkerBaseListenPort)+int(index), - ), - ) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - _, addr, err := mn.DialArgs(ma) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - ctx, cancel := context.WithTimeout(e.ctx, 1*time.Second) - defer cancel() - conn, err := qgrpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials( - insecure.NewCredentials(), - ), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(10*1024*1024), - grpc.MaxCallRecvMsgSize(10*1024*1024), - ), - grpc.WithBlock(), - ) - if err != nil { - return nil, errors.Wrap(err, "create parallel data client") - } - - client := protobufs.NewDataIPCServiceClient(conn) - - e.logger.Info( - "connected to data worker process", - zap.Uint32("client", index), - ) - return client, nil -} - -func (e *DataClockConsensusEngine) createParallelDataClientsFromList() ( - []protobufs.DataIPCServiceClient, - error, -) { - parallelism := len(e.config.Engine.DataWorkerMultiaddrs) - - e.logger.Info( - "connecting to data worker processes", - zap.Int("parallelism", parallelism), - ) - - clients := make([]protobufs.DataIPCServiceClient, parallelism) - - for i := 0; i < parallelism; i++ { - ma, err := multiaddr.NewMultiaddr(e.config.Engine.DataWorkerMultiaddrs[i]) - if err != nil { - panic(err) - } - - _, addr, err := mn.DialArgs(ma) - if err != nil { - e.logger.Error("could not get dial args", zap.Error(err)) - continue - } - - ctx, cancel := context.WithTimeout(e.ctx, 1*time.Second) - defer cancel() - conn, err := qgrpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials( - insecure.NewCredentials(), - ), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(10*1024*1024), - grpc.MaxCallRecvMsgSize(10*1024*1024), - ), - grpc.WithBlock(), - ) - if err != nil { - e.logger.Error("could not dial", zap.Error(err)) - continue - } - - clients[i] = protobufs.NewDataIPCServiceClient(conn) - } - - e.logger.Info( - "connected to data worker processes", - zap.Int("parallelism", parallelism), - ) - return clients, nil -} - -func (e *DataClockConsensusEngine) createParallelDataClientsFromBaseMultiaddr( - parallelism int, -) ([]protobufs.DataIPCServiceClient, error) { - e.logger.Info( - "connecting to data worker processes", - zap.Int("parallelism", parallelism), - ) - - clients := make([]protobufs.DataIPCServiceClient, parallelism) - - for i := 0; i < parallelism; i++ { - ma, err := multiaddr.NewMultiaddr( + var ma multiaddr.Multiaddr + var err error + if useList { + ma, err = multiaddr.NewMultiaddr(e.config.Engine.DataWorkerMultiaddrs[index]) + } else { + ma, err = multiaddr.NewMultiaddr( fmt.Sprintf( e.config.Engine.DataWorkerBaseListenMultiaddr, - int(e.config.Engine.DataWorkerBaseListenPort)+i, + int(e.config.Engine.DataWorkerBaseListenPort)+int(index), ), ) - if err != nil { - panic(err) - } + } + if err != nil { + e.logger.Error("failed to create multiaddr", zap.Error(err)) + return nil, err + } - _, addr, err := mn.DialArgs(ma) - if err != nil { - e.logger.Error("could not get dial args", zap.Error(err)) - continue - } - ctx, cancel := context.WithTimeout(e.ctx, 1*time.Second) - defer cancel() - conn, err := qgrpc.DialContext( - ctx, - addr, - grpc.WithTransportCredentials( - insecure.NewCredentials(), - ), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(10*1024*1024), - grpc.MaxCallRecvMsgSize(10*1024*1024), - ), - grpc.WithBlock(), + _, addr, err := mn.DialArgs(ma) + + if err != nil { + e.logger.Error("could not get dial args", + zap.Error(err), + zap.String("multiaddr", ma.String()), + zap.Int("index", index), ) - if err != nil { - e.logger.Error("could not dial", zap.Error(err)) - continue - } + return nil, err + } - clients[i] = protobufs.NewDataIPCServiceClient(conn) + ctx, cancel := context.WithTimeout(e.ctx, 1*time.Second) + defer cancel() + conn, err := qgrpc.DialContext( + ctx, + addr, + grpc.WithTransportCredentials( + insecure.NewCredentials(), + ), + grpc.WithDefaultCallOptions( + grpc.MaxCallSendMsgSize(10*1024*1024), + grpc.MaxCallRecvMsgSize(10*1024*1024), + ), + grpc.WithBlock(), + ) + if err != nil { + e.logger.Error("could not dial", + zap.Error(err), + zap.String("multiaddr", ma.String()), + zap.Int("index", index), + ) + return nil, err } e.logger.Info( - "connected to data worker processes", + "connected to data worker process", + zap.String("multiaddr", ma.String()), + ) + + return protobufs.NewDataIPCServiceClient(conn), nil + +} + +func (e *DataClockConsensusEngine) createParallelDataWorkerClients() { + parallelism := len(e.config.Engine.DataWorkerMultiaddrs) + useList := true + if parallelism == 0 { + parallelism = e.config.Engine.DataWorkerCount + useList = false + } + + e.clientsMx.Lock() + e.clients = make([]protobufs.DataIPCServiceClient, parallelism) + e.clientsMx.Unlock() + + e.logger.Info( + "connecting to data worker processes", zap.Int("parallelism", parallelism), ) - return clients, nil + + wg := sync.WaitGroup{} + wg.Add(parallelism) + for i := 0; i < parallelism; i++ { + index := i + go func() { + defer wg.Done() + client, err := e.connectToClient(index, useList) + if err != nil { + e.clientsMx.Lock() + e.clients[index] = nil + e.clientsMx.Unlock() + e.logger.Error("failed to connect to data worker", zap.Error(err)) + return + } + e.clientsMx.Lock() + e.clients[index] = client + e.clientsMx.Unlock() + }() + } + wg.Wait() +} + +func (e *DataClockConsensusEngine) tryReconnectDataWorkerClients() { + // could reload worker list config here + parallelism := len(e.config.Engine.DataWorkerMultiaddrs) + useList := true + if parallelism == 0 { + parallelism = e.config.Engine.DataWorkerCount + useList = false + } + + wg := sync.WaitGroup{} + wg.Add(parallelism) + for i := 0; i < parallelism; i++ { + index := i + + go func() { + defer wg.Done() + if e.clients[index] != nil { + return + } + for j := 3; j >= 0; j-- { + client, err := e.connectToClient(index, useList) + if err != nil { + e.clientsMx.Lock() + e.clients[index] = nil + e.clientsMx.Unlock() + e.logger.Error("failed to connect to data worker", + zap.Error(err), + zap.Int("index", index), + ) + time.Sleep(50 * time.Millisecond) + continue + } + e.clientsMx.Lock() + e.logger.Info("reconnected to data worker", + zap.Int("index", index), + ) + e.clients[index] = client + e.clientsMx.Unlock() + break + } + }() + } + wg.Wait() } func (e *DataClockConsensusEngine) GetWorkerCount() uint32 { diff --git a/node/consensus/data/main_data_loop.go b/node/consensus/data/main_data_loop.go index d82f2b8..15f8158 100644 --- a/node/consensus/data/main_data_loop.go +++ b/node/consensus/data/main_data_loop.go @@ -2,10 +2,8 @@ package data import ( "bytes" - "sync" "time" - "github.com/iden3/go-iden3-crypto/poseidon" "go.uber.org/zap" "source.quilibrium.com/quilibrium/monorepo/node/consensus" "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" @@ -233,6 +231,7 @@ func (e *DataClockConsensusEngine) processFrame( latestFrame *protobufs.ClockFrame, dataFrame *protobufs.ClockFrame, ) *protobufs.ClockFrame { + e.logger.Info( "current frame head", zap.Uint64("frame_number", dataFrame.FrameNumber), @@ -275,175 +274,12 @@ func (e *DataClockConsensusEngine) processFrame( return dataFrame } - e.dataTimeReel.Insert(e.ctx, nextFrame, true) + if _, err := e.dataTimeReel.Insert(e.ctx, nextFrame); err != nil { + e.logger.Debug("could not insert frame", zap.Error(err)) + } return nextFrame } else { - if latestFrame.Timestamp > time.Now().UnixMilli()-120000 { - if !e.IsInProverTrie(e.pubSub.GetPeerID()) { - e.logger.Info("announcing prover join") - for _, eng := range e.executionEngines { - eng.AnnounceProverJoin() - break - } - } else { - if e.previousFrameProven != nil && - e.previousFrameProven.FrameNumber == latestFrame.FrameNumber { - return latestFrame - } - - h, err := poseidon.HashBytes(e.pubSub.GetPeerID()) - if err != nil { - panic(err) - } - peerProvingKeyAddress := h.FillBytes(make([]byte, 32)) - - ring := -1 - if tries := e.GetFrameProverTries(); len(tries) > 1 { - for i, tries := range tries[1:] { - i := i - if tries.Contains(peerProvingKeyAddress) { - ring = i - } - } - } - - e.clientReconnectTest++ - if e.clientReconnectTest >= 10 { - wg := sync.WaitGroup{} - wg.Add(len(e.clients)) - for i, client := range e.clients { - i := i - client := client - go func() { - for j := 3; j >= 0; j-- { - var err error - if client == nil { - if len(e.config.Engine.DataWorkerMultiaddrs) != 0 { - e.logger.Error( - "client failed, reconnecting after 50ms", - zap.Uint32("client", uint32(i)), - ) - time.Sleep(50 * time.Millisecond) - client, err = e.createParallelDataClientsFromListAndIndex(uint32(i)) - if err != nil { - e.logger.Error("failed to reconnect", zap.Error(err)) - } - } else if len(e.config.Engine.DataWorkerMultiaddrs) == 0 { - e.logger.Error( - "client failed, reconnecting after 50ms", - zap.Uint32("client", uint32(i)), - ) - time.Sleep(50 * time.Millisecond) - client, err = - e.createParallelDataClientsFromBaseMultiaddrAndIndex(uint32(i)) - if err != nil { - e.logger.Error( - "failed to reconnect", - zap.Uint32("client", uint32(i)), - zap.Error(err), - ) - } - } - e.clients[i] = client - continue - } - } - wg.Done() - }() - } - wg.Wait() - e.clientReconnectTest = 0 - } - - outputs := e.PerformTimeProof(latestFrame, latestFrame.Difficulty, ring) - if outputs == nil || len(outputs) < 3 { - e.logger.Info("workers not yet available for proving") - return latestFrame - } - modulo := len(outputs) - proofTree, output, err := tries.PackOutputIntoPayloadAndProof( - outputs, - modulo, - latestFrame, - e.previousTree, - ) - if err != nil { - e.logger.Error( - "could not successfully pack proof, reattempting", - zap.Error(err), - ) - return latestFrame - } - e.previousFrameProven = latestFrame - e.previousTree = proofTree - - mint := &protobufs.MintCoinRequest{ - Proofs: output, - } - if err := mint.SignED448(e.pubSub.GetPublicKey(), e.pubSub.SignMessage); err != nil { - e.logger.Error("could not sign mint", zap.Error(err)) - return latestFrame - } - if err := mint.Validate(); err != nil { - e.logger.Error("mint validation failed", zap.Error(err)) - return latestFrame - } - - e.logger.Info( - "submitting data proof", - zap.Int("ring", ring), - zap.Int("active_workers", len(outputs)), - zap.Uint64("frame_number", latestFrame.FrameNumber), - zap.Duration("frame_age", frametime.Since(latestFrame)), - ) - - if err := e.publishMessage(e.txFilter, mint.TokenRequest()); err != nil { - e.logger.Error("could not publish mint", zap.Error(err)) - } - - if e.config.Engine.AutoMergeCoins { - _, addrs, _, err := e.coinStore.GetCoinsForOwner( - peerProvingKeyAddress, - ) - if err != nil { - e.logger.Error( - "received error while iterating coins", - zap.Error(err), - ) - return latestFrame - } - - if len(addrs) > 25 { - refs := []*protobufs.CoinRef{} - for _, addr := range addrs { - refs = append(refs, &protobufs.CoinRef{ - Address: addr, - }) - } - - merge := &protobufs.MergeCoinRequest{ - Coins: refs, - } - if err := merge.SignED448( - e.pubSub.GetPublicKey(), - e.pubSub.SignMessage, - ); err != nil { - e.logger.Error("could not sign merge", zap.Error(err)) - return latestFrame - } - if err := merge.Validate(); err != nil { - e.logger.Error("merge validation failed", zap.Error(err)) - return latestFrame - } - - if err := e.publishMessage(e.txFilter, merge.TokenRequest()); err != nil { - e.logger.Warn("could not publish merge", zap.Error(err)) - } - } - } - } - } return latestFrame } } diff --git a/node/consensus/data/message_handler.go b/node/consensus/data/message_handler.go index 23e856a..2020b82 100644 --- a/node/consensus/data/message_handler.go +++ b/node/consensus/data/message_handler.go @@ -227,6 +227,12 @@ func (e *DataClockConsensusEngine) handleClockFrame( return nil } + if _, ok := e.recentlyProcessedFrames.Peek(string(frame.Output)); ok { + return nil + } + + e.recentlyProcessedFrames.Add(string(frame.Output), struct{}{}) + e.logger.Debug( "got clock frame", zap.Binary("address", address), @@ -253,7 +259,11 @@ func (e *DataClockConsensusEngine) handleClockFrame( } if frame.FrameNumber > head.FrameNumber { - e.dataTimeReel.Insert(e.ctx, frame, false) + go e.initiateProvers(frame) + + if _, err := e.dataTimeReel.Insert(e.ctx, frame); err != nil { + e.logger.Debug("could not insert frame", zap.Error(err)) + } } return nil diff --git a/node/consensus/master/broadcast_messaging.go b/node/consensus/master/broadcast_messaging.go index cde9b3f..9119d7d 100644 --- a/node/consensus/master/broadcast_messaging.go +++ b/node/consensus/master/broadcast_messaging.go @@ -155,7 +155,7 @@ func (e *MasterClockConsensusEngine) publishProof( zap.Uint64("frame_number", frame.FrameNumber), ) - e.masterTimeReel.Insert(context.TODO(), frame, false) + e.masterTimeReel.Insert(context.TODO(), frame) } e.state = consensus.EngineStateCollecting diff --git a/node/consensus/master/master_clock_consensus_engine.go b/node/consensus/master/master_clock_consensus_engine.go index 54fa8f7..e7ea32f 100644 --- a/node/consensus/master/master_clock_consensus_engine.go +++ b/node/consensus/master/master_clock_consensus_engine.go @@ -208,7 +208,7 @@ func (e *MasterClockConsensusEngine) Start() <-chan error { continue } - e.masterTimeReel.Insert(context.TODO(), newFrame, false) + e.masterTimeReel.Insert(context.TODO(), newFrame) } } }() diff --git a/node/consensus/time/data_time_reel.go b/node/consensus/time/data_time_reel.go index 25e08f7..f5f9ec4 100644 --- a/node/consensus/time/data_time_reel.go +++ b/node/consensus/time/data_time_reel.go @@ -29,6 +29,7 @@ type pendingFrame struct { selector *big.Int parentSelector *big.Int frameNumber uint64 + done chan struct{} } type DataTimeReel struct { @@ -190,12 +191,18 @@ func (d *DataTimeReel) Head() (*protobufs.ClockFrame, error) { return d.head, nil } +var alreadyDone chan struct{} = func() chan struct{} { + done := make(chan struct{}) + close(done) + return done +}() + // Insert enqueues a structurally valid frame into the time reel. If the frame // is the next one in sequence, it advances the reel head forward and emits a // new frame on the new frame channel. -func (d *DataTimeReel) Insert(ctx context.Context, frame *protobufs.ClockFrame, isSync bool) error { +func (d *DataTimeReel) Insert(ctx context.Context, frame *protobufs.ClockFrame) (<-chan struct{}, error) { if err := d.ctx.Err(); err != nil { - return err + return nil, err } d.logger.Debug( @@ -222,21 +229,24 @@ func (d *DataTimeReel) Insert(ctx context.Context, frame *protobufs.ClockFrame, d.storePending(selector, parent, distance, frame) if d.head.FrameNumber+1 == frame.FrameNumber { + done := make(chan struct{}) select { case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() case <-d.ctx.Done(): - return d.ctx.Err() + return nil, d.ctx.Err() case d.frames <- &pendingFrame{ selector: selector, parentSelector: parent, frameNumber: frame.FrameNumber, + done: done, }: + return done, nil } } } - return nil + return alreadyDone, nil } func ( @@ -393,6 +403,7 @@ func (d *DataTimeReel) runLoop() { // Otherwise set it as the next and process all pending if err = d.setHead(rawFrame, distance); err != nil { + close(frame.done) continue } d.processPending(d.head, frame) @@ -559,6 +570,7 @@ func (d *DataTimeReel) processPending( frame *protobufs.ClockFrame, lastReceived *pendingFrame, ) { + defer close(lastReceived.done) // d.logger.Debug( // "process pending", // zap.Uint64("head_frame", frame.FrameNumber), diff --git a/node/consensus/time/data_time_reel_test.go b/node/consensus/time/data_time_reel_test.go index 43cc07d..3ebce28 100644 --- a/node/consensus/time/data_time_reel_test.go +++ b/node/consensus/time/data_time_reel_test.go @@ -233,7 +233,7 @@ func TestDataTimeReel(t *testing.T) { i+1, 10, ) - d.Insert(ctx, frame, false) + d.Insert(ctx, frame) prevBI, _ := frame.GetSelector() prev = prevBI.FillBytes(make([]byte, 32)) } @@ -264,7 +264,7 @@ func TestDataTimeReel(t *testing.T) { } for i := 99; i >= 0; i-- { - err := d.Insert(ctx, insertFrames[i], false) + _, err := d.Insert(ctx, insertFrames[i]) assert.NoError(t, err) } @@ -286,7 +286,7 @@ func TestDataTimeReel(t *testing.T) { i+1, 10, ) - d.Insert(ctx, frame, false) + d.Insert(ctx, frame) prevBI, _ := frame.GetSelector() prev = prevBI.FillBytes(make([]byte, 32)) @@ -334,7 +334,7 @@ func TestDataTimeReel(t *testing.T) { } for i := 99; i >= 0; i-- { - err := d.Insert(ctx, insertFrames[i], false) + _, err := d.Insert(ctx, insertFrames[i]) assert.NoError(t, err) } @@ -397,7 +397,7 @@ func TestDataTimeReel(t *testing.T) { // Someone is honest, but running backwards: for i := 99; i >= 0; i-- { - err := d.Insert(ctx, insertFrames[i], false) + _, err := d.Insert(ctx, insertFrames[i]) gotime.Sleep(1 * gotime.Second) assert.NoError(t, err) } diff --git a/node/consensus/time/master_time_reel.go b/node/consensus/time/master_time_reel.go index 70ef4d3..fcfe0d6 100644 --- a/node/consensus/time/master_time_reel.go +++ b/node/consensus/time/master_time_reel.go @@ -123,13 +123,12 @@ func (m *MasterTimeReel) Head() (*protobufs.ClockFrame, error) { func (m *MasterTimeReel) Insert( ctx context.Context, frame *protobufs.ClockFrame, - isSync bool, -) error { +) (<-chan struct{}, error) { go func() { m.frames <- frame }() - return nil + return alreadyDone, nil } // NewFrameCh implements TimeReel. diff --git a/node/consensus/time/master_time_reel_test.go b/node/consensus/time/master_time_reel_test.go index 6332d3f..4184598 100644 --- a/node/consensus/time/master_time_reel_test.go +++ b/node/consensus/time/master_time_reel_test.go @@ -61,7 +61,7 @@ func TestMasterTimeReel(t *testing.T) { ) assert.NoError(t, err) - err := m.Insert(ctx, frame, false) + _, err := m.Insert(ctx, frame) assert.NoError(t, err) } @@ -81,7 +81,7 @@ func TestMasterTimeReel(t *testing.T) { } for i := 99; i >= 0; i-- { - err := m.Insert(ctx, insertFrames[i], false) + _, err := m.Insert(ctx, insertFrames[i]) assert.NoError(t, err) } diff --git a/node/consensus/time/time_reel.go b/node/consensus/time/time_reel.go index 489b892..26719d2 100644 --- a/node/consensus/time/time_reel.go +++ b/node/consensus/time/time_reel.go @@ -9,7 +9,7 @@ import ( type TimeReel interface { Start() error Stop() - Insert(ctx context.Context, frame *protobufs.ClockFrame, isSync bool) error + Insert(ctx context.Context, frame *protobufs.ClockFrame) (<-chan struct{}, error) Head() (*protobufs.ClockFrame, error) NewFrameCh() <-chan *protobufs.ClockFrame BadFrameCh() <-chan *protobufs.ClockFrame diff --git a/node/execution/intrinsics/token/application/token_application.go b/node/execution/intrinsics/token/application/token_application.go index 9126d47..d6e4c54 100644 --- a/node/execution/intrinsics/token/application/token_application.go +++ b/node/execution/intrinsics/token/application/token_application.go @@ -213,7 +213,7 @@ func (a *TokenApplication) ApplyTransitions( fails[i] = transition continue } - ring, parallelism, err := t.Mint.RingAndParallelism( + _, _, err := t.Mint.RingAndParallelism( func(addr []byte) int { if _, ok := seen[string(addr)]; ok { return -1 @@ -233,7 +233,6 @@ func (a *TokenApplication) ApplyTransitions( if err == nil { // fmt.Println(i, "checked ring test") set[i] = transition - parallelismMap[ring] = parallelismMap[ring] + uint64(parallelism) } else { // fmt.Println(i, "failed ring test", err) fails[i] = transition @@ -246,6 +245,7 @@ func (a *TokenApplication) ApplyTransitions( outputsSet := make([][]*protobufs.TokenOutput, len(set)) successes := make([]*protobufs.TokenRequest, len(set)) + processedMap := make([]*processedMint, len(set)) for i, transition := range set { if transition == nil { continue @@ -390,6 +390,74 @@ func (a *TokenApplication) ApplyTransitions( if transition == nil { continue } + switch t := transition.Request.(type) { + case *protobufs.TokenRequest_Mint: + throttle <- struct{}{} + wg.Add(1) + go func(i int, transition *protobufs.TokenRequest) { + defer func() { <-throttle }() + defer wg.Done() + var err error + processedMap[i], err = a.preProcessMint( + currentFrameNumber, + t.Mint, + frame, + ) + if err != nil { + fails[i] = transition + return + } + }(i, transition) + } + } + + wg.Wait() + + for i, transition := range set { + if fails[i] != nil { + continue + } + switch t := transition.Request.(type) { + case *protobufs.TokenRequest_Mint: + if len(t.Mint.Proofs) == 1 { + continue + } else if len(t.Mint.Proofs) >= 3 && currentFrameNumber > PROOF_FRAME_CUTOFF { + if processedMap[i].validForReward { + ring, parallelism, err := t.Mint.RingAndParallelism( + func(addr []byte) int { + ring := -1 + for i, t := range a.Tries[1:] { + if t.Contains(addr) { + ring = i + break + } + } + + return ring + }, + ) + if err == nil { + parallelismMap[ring] = parallelismMap[ring] + uint64(parallelism) + } else { + // fmt.Println(i, "failed ring test", err) + fails[i] = transition + } + } + } + default: + set[i] = transition + } + } + + for i, transition := range set { + if transition == nil { + continue + } + + if fails[i] != nil { + continue + } + switch t := transition.Request.(type) { case *protobufs.TokenRequest_Mint: throttle <- struct{}{} @@ -401,6 +469,7 @@ func (a *TokenApplication) ApplyTransitions( currentFrameNumber, t.Mint, frame, + processedMap[i], parallelismMap, ) if err != nil { @@ -412,6 +481,7 @@ func (a *TokenApplication) ApplyTransitions( }(i, transition) } } + wg.Wait() finalFails := []*protobufs.TokenRequest{} @@ -422,7 +492,7 @@ func (a *TokenApplication) ApplyTransitions( } if len(finalFails) != 0 && !skipFailures { return nil, nil, nil, errors.Wrap( - err, + ErrInvalidStateTransition, "apply transitions", ) } @@ -446,7 +516,6 @@ func (a *TokenApplication) ApplyTransitions( finalizedTransitions.Requests = finalSuccesses failedTransitions.Requests = finalFails - return a, finalizedTransitions, failedTransitions, nil } diff --git a/node/execution/intrinsics/token/application/token_handle_mint.go b/node/execution/intrinsics/token/application/token_handle_mint.go index dfb23ee..0bf2020 100644 --- a/node/execution/intrinsics/token/application/token_handle_mint.go +++ b/node/execution/intrinsics/token/application/token_handle_mint.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "math/big" + "math/bits" "github.com/iden3/go-iden3-crypto/poseidon" pcrypto "github.com/libp2p/go-libp2p/core/crypto" @@ -18,22 +19,473 @@ import ( ) // for tests, these need to be var -const PROOF_FRAME_CUTOFF = 1 -const PROOF_FRAME_RING_RESET = 5750 -const PROOF_FRAME_RING_RESET_2 = 7650 -const PROOF_FRAME_RING_RESET_3 = 13369 -const PROOF_FRAME_SENIORITY_REPAIR = 25745 +var PROOF_FRAME_CUTOFF = uint64(46500) +var PROOF_FRAME_RING_RESET = uint64(52000) +var PROOF_FRAME_RING_RESET_2 = uint64(53028) +var PROOF_FRAME_COMBINE_CUTOFF = uint64(162000) + +const PROOF_FRAME_SENIORITY_REPAIR = 59029 + +type processedMint struct { + isPre2 bool + penalty bool + deletedProof *protobufs.TokenOutput_DeletedProof + parallelism uint32 + priorCommitment []byte + newCommitment []byte + newFrameNumber uint64 + implicitAddr []byte + validForReward bool + treeVerified bool + wesoVerified bool +} + +func (a *TokenApplication) preProcessMint( + currentFrameNumber uint64, + t *protobufs.MintCoinRequest, + frame *protobufs.ClockFrame, +) ( + out *processedMint, + err error, +) { + if err := t.Validate(); err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + + payload := []byte("mint") + for _, p := range t.Proofs { + payload = append(payload, p...) + } + + pk, err := pcrypto.UnmarshalEd448PublicKey( + t.Signature.PublicKey.KeyValue, + ) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + + peerId, err := peer.IDFromPublicKey(pk) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + + addr, err := poseidon.HashBytes( + t.Signature.PublicKey.KeyValue, + ) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + addrBytes := addr.FillBytes(make([]byte, 32)) + + altAddr, err := poseidon.HashBytes([]byte(peerId)) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + altAddrBytes := altAddr.FillBytes(make([]byte, 32)) + + // todo: set termination frame for this: + if len(t.Proofs) == 1 && a.Tries[0].Contains(addrBytes) && + bytes.Equal(t.Signature.PublicKey.KeyValue, a.Beacon) { + return &processedMint{ + isPre2: true, + penalty: false, + }, nil + } else if len(t.Proofs) > 0 && currentFrameNumber > PROOF_FRAME_CUTOFF && + currentFrameNumber < PROOF_FRAME_COMBINE_CUTOFF { + a.Logger.Debug( + "got mint from peer", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + _, prfs, err := a.CoinStore.GetPreCoinProofsForOwner(altAddrBytes) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + + var delete *protobufs.PreCoinProof + var commitment []byte + var previousFrame *protobufs.ClockFrame + var previousParallelism uint32 + for _, pr := range prfs { + if len(pr.Proof) >= 3 && (len(pr.Commitment) == 40 || len(pr.Commitment) == 72) { + delete = pr + commitment = pr.Commitment[:32] + previousFrameNumber := binary.BigEndian.Uint64(pr.Commitment[32:]) + previousParallelism = binary.BigEndian.Uint32(pr.Proof[36:40]) + previousFrame, _, err = a.ClockStore.GetDataClockFrame( + frame.Filter, + previousFrameNumber, + true, + ) + + if err != nil { + a.Logger.Debug( + "invalid frame", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + } + } + + newCommitment, parallelism, newFrameNumber, verified, err := + tries.UnpackAndVerifyOutput(commitment, t.Proofs) + if err != nil { + a.Logger.Debug( + "mint error", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + + if previousParallelism != 0 && previousParallelism != parallelism { + verified = false + } + + if !verified { + a.Logger.Debug( + "tree verification failed", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + } + + // Current frame - 2 is because the current frame is the newly created frame, + // and the provers are submitting proofs on the frame preceding the one they + // last saw. This enforces liveness and creates a punishment for being + // late. + if (previousFrame != nil && newFrameNumber <= previousFrame.FrameNumber) || + newFrameNumber < currentFrameNumber-2 { + previousFrameNumber := uint64(0) + if previousFrame != nil { + previousFrameNumber = previousFrame.FrameNumber + } + + a.Logger.Debug( + "received out of order proofs, ignoring", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("previous_frame", previousFrameNumber), + zap.Uint64("new_frame", newFrameNumber), + zap.Uint64("frame_number", currentFrameNumber), + ) + return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") + } + + wesoVerified := true + if verified && delete != nil && len(t.Proofs) > 3 { + newFrame, _, err := a.ClockStore.GetDataClockFrame( + frame.Filter, + newFrameNumber, + true, + ) + if err != nil { + a.Logger.Debug( + "invalid frame", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + hash := sha3.Sum256(newFrame.Output) + pick := tries.BytesToUnbiasedMod(hash, uint64(parallelism)) + challenge := []byte{} + challenge = append(challenge, peerId...) + challenge = binary.BigEndian.AppendUint64( + challenge, + previousFrame.FrameNumber, + ) + individualChallenge := append([]byte{}, challenge...) + individualChallenge = binary.BigEndian.AppendUint32( + individualChallenge, + uint32(pick), + ) + leaf := t.Proofs[len(t.Proofs)-1] + individualChallenge = append(individualChallenge, previousFrame.Output...) + if len(leaf) != 516 { + a.Logger.Debug( + "invalid size", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + zap.Int("proof_size", len(leaf)), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + + if bytes.Equal(leaf, bytes.Repeat([]byte{0x00}, 516)) || + !a.FrameProver.VerifyChallengeProof( + individualChallenge, + frame.Difficulty, + leaf, + ) { + a.Logger.Debug( + "invalid proof", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + // we want this to still apply the next commit even if this proof failed + wesoVerified = false + } + } + + var deletedProof *protobufs.TokenOutput_DeletedProof + if delete != nil { + deletedProof = &protobufs.TokenOutput_DeletedProof{ + DeletedProof: delete, + } + } + + validForReward := verified && delete != nil && len(t.Proofs) > 3 && wesoVerified + return &processedMint{ + isPre2: false, + penalty: false, + deletedProof: deletedProof, + parallelism: parallelism, + newCommitment: newCommitment, + newFrameNumber: newFrameNumber, + implicitAddr: altAddrBytes, + validForReward: validForReward, + treeVerified: verified, + wesoVerified: wesoVerified, + }, nil + } else if len(t.Proofs) > 0 && currentFrameNumber >= PROOF_FRAME_COMBINE_CUTOFF { + a.Logger.Debug( + "got mint from peer", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + _, prfs, err := a.CoinStore.GetPreCoinProofsForOwner(altAddrBytes) + if err != nil { + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") + } + + var delete *protobufs.PreCoinProof + var commitment []byte + var previousFrame *protobufs.ClockFrame + var previousParallelism uint32 + var priorCommitment []byte + for _, pr := range prfs { + if len(pr.Proof) >= 3 && (len(pr.Commitment) == 40 || len(pr.Commitment) == 72) { + delete = pr + commitment = pr.Commitment[:32] + previousFrameNumber := binary.BigEndian.Uint64(pr.Commitment[32:40]) + previousParallelism = binary.BigEndian.Uint32(pr.Proof[36:40]) + previousFrame, _, err = a.ClockStore.GetDataClockFrame( + frame.Filter, + previousFrameNumber, + true, + ) + if len(pr.Commitment) > 40 { + priorCommitment = pr.Commitment[40:] + } + + if err != nil { + a.Logger.Debug( + "invalid frame", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + } + } + + newCommitment, parallelism, newFrameNumber, verified, err := + tries.UnpackAndVerifyMultiOutput(commitment, t.Proofs) + if err != nil { + a.Logger.Debug( + "mint error", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + + if previousParallelism != 0 && previousParallelism != parallelism { + verified = false + } + + if !verified { + a.Logger.Debug( + "tree verification failed", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + } + + // Current frame - 2 is because the current frame is the newly created frame, + // and the provers are submitting proofs on the frame preceding the one they + // last saw. This enforces liveness and creates a punishment for being + // late. + if (previousFrame != nil && newFrameNumber <= previousFrame.FrameNumber) || + newFrameNumber < currentFrameNumber-2 { + previousFrameNumber := uint64(0) + if previousFrame != nil { + previousFrameNumber = previousFrame.FrameNumber + } + + a.Logger.Debug( + "received out of order proofs, ignoring", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("previous_frame", previousFrameNumber), + zap.Uint64("new_frame", newFrameNumber), + zap.Uint64("frame_number", currentFrameNumber), + ) + return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") + } + + wesoVerified := true + if verified && delete != nil && len(t.Proofs) > 3 { + newFrame, _, err := a.ClockStore.GetDataClockFrame( + frame.Filter, + newFrameNumber, + true, + ) + if err != nil { + a.Logger.Debug( + "invalid frame", + zap.Error(err), + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + hash := sha3.Sum256(append(append([]byte{}, newFrame.Output...), commitment...)) + pick := tries.BytesToUnbiasedMod(hash, uint64(parallelism)) + challenge := []byte{} + challenge = append(challenge, peerId...) + challenge = binary.BigEndian.AppendUint64( + challenge, + previousFrame.FrameNumber, + ) + additional := bits.Len64(uint64(parallelism)-1) - 1 + picks := []int{int(pick)} + outputs := [][]byte{t.Proofs[len(t.Proofs)-(additional+1)]} + for additional > 0 { + hash = sha3.Sum256(hash[:]) + pick := tries.BytesToUnbiasedMod(hash, uint64(parallelism)) + found := false + for _, p := range picks { + if p == int(pick) { + found = true + break + } + } + + if !found { + picks = append(picks, int(pick)) + outputs = append(outputs, t.Proofs[len(t.Proofs)-additional]) + additional-- + } + } + for i, pick := range picks { + individualChallenge := append([]byte{}, challenge...) + individualChallenge = binary.BigEndian.AppendUint32( + individualChallenge, + uint32(pick), + ) + + individualChallenge = append(individualChallenge, previousFrame.Output...) + individualChallenge = append(individualChallenge, priorCommitment...) + leaf := outputs[i] + + if len(leaf) != 516 { + a.Logger.Debug( + "invalid size", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + zap.Int("proof_size", len(leaf)), + ) + return &processedMint{ + isPre2: false, + penalty: true, + }, nil + } + + if bytes.Equal(leaf, bytes.Repeat([]byte{0x00}, 516)) || + !a.FrameProver.VerifyChallengeProof( + individualChallenge, + frame.Difficulty, + leaf, + ) { + a.Logger.Debug( + "invalid proof", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + // we want this to still apply the next commit even if this proof failed + wesoVerified = wesoVerified && false + } + } + } + + var deletedProof *protobufs.TokenOutput_DeletedProof + if delete != nil { + deletedProof = &protobufs.TokenOutput_DeletedProof{ + DeletedProof: delete, + } + } + + validForReward := verified && delete != nil && len(t.Proofs) > 3 && wesoVerified + return &processedMint{ + isPre2: false, + penalty: false, + deletedProof: deletedProof, + parallelism: parallelism, + priorCommitment: commitment, + newCommitment: newCommitment, + newFrameNumber: newFrameNumber, + implicitAddr: altAddrBytes, + validForReward: validForReward, + treeVerified: verified, + wesoVerified: wesoVerified, + }, nil + } + + a.Logger.Debug( + "could not find case for proof", + zap.String("peer_id", base58.Encode([]byte(peerId))), + zap.Uint64("frame_number", currentFrameNumber), + ) + return nil, errors.Wrap(ErrInvalidStateTransition, "pre process mint") +} func (a *TokenApplication) handleMint( currentFrameNumber uint64, t *protobufs.MintCoinRequest, frame *protobufs.ClockFrame, + processed *processedMint, parallelismMap map[int]uint64, ) ([]*protobufs.TokenOutput, error) { - if err := t.Validate(); err != nil { - return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") - } - payload := []byte("mint") for _, p := range t.Proofs { payload = append(payload, p...) @@ -51,14 +503,6 @@ func (a *TokenApplication) handleMint( return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") } - addr, err := poseidon.HashBytes( - t.Signature.PublicKey.KeyValue, - ) - if err != nil { - return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") - } - addrBytes := addr.FillBytes(make([]byte, 32)) - altAddr, err := poseidon.HashBytes([]byte(peerId)) if err != nil { return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") @@ -66,8 +510,7 @@ func (a *TokenApplication) handleMint( altAddrBytes := altAddr.FillBytes(make([]byte, 32)) // todo: set termination frame for this: - if len(t.Proofs) == 1 && a.Tries[0].Contains(addrBytes) && - bytes.Equal(t.Signature.PublicKey.KeyValue, a.Beacon) { + if processed.isPre2 { if len(t.Proofs[0]) != 64 { return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") } @@ -131,60 +574,7 @@ func (a *TokenApplication) handleMint( } } - _, prfs, err := a.CoinStore.GetPreCoinProofsForOwner(altAddrBytes) - if err != nil { - return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") - } - - var delete *protobufs.PreCoinProof - var commitment []byte - var previousFrame *protobufs.ClockFrame - for _, pr := range prfs { - if len(pr.Proof) >= 3 && len(pr.Commitment) == 40 { - delete = pr - commitment = pr.Commitment[:32] - previousFrameNumber := binary.BigEndian.Uint64(pr.Commitment[32:]) - previousFrame, _, err = a.ClockStore.GetDataClockFrame( - frame.Filter, - previousFrameNumber, - true, - ) - - if err != nil { - a.Logger.Debug( - "invalid frame", - zap.Error(err), - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - ) - return []*protobufs.TokenOutput{&protobufs.TokenOutput{ - Output: &protobufs.TokenOutput_Penalty{ - Penalty: &protobufs.ProverPenalty{ - Quantity: 10, - Account: &protobufs.AccountRef{ - Account: &protobufs.AccountRef_ImplicitAccount{ - ImplicitAccount: &protobufs.ImplicitAccount{ - ImplicitType: 0, - Address: altAddrBytes, - }, - }, - }, - }, - }, - }}, nil - } - } - } - - newCommitment, parallelism, newFrameNumber, verified, err := - tries.UnpackAndVerifyOutput(commitment, t.Proofs) - if err != nil { - a.Logger.Debug( - "mint error", - zap.Error(err), - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - ) + if processed.penalty { return []*protobufs.TokenOutput{&protobufs.TokenOutput{ Output: &protobufs.TokenOutput_Penalty{ Penalty: &protobufs.ProverPenalty{ @@ -202,140 +592,24 @@ func (a *TokenApplication) handleMint( }}, nil } - if !verified { - a.Logger.Debug( - "tree verification failed", - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - ) - } - - // Current frame - 2 is because the current frame is the newly created frame, - // and the provers are submitting proofs on the frame preceding the one they - // last saw. This enforces liveness and creates a punishment for being - // late. - if (previousFrame != nil && newFrameNumber <= previousFrame.FrameNumber) || - newFrameNumber < currentFrameNumber-2 { - previousFrameNumber := uint64(0) - if previousFrame != nil { - previousFrameNumber = previousFrame.FrameNumber - } - a.Logger.Debug( - "received out of order proofs, ignoring", - zap.Error(err), - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("previous_frame", previousFrameNumber), - zap.Uint64("new_frame", newFrameNumber), - zap.Uint64("frame_number", currentFrameNumber), - ) - return nil, errors.Wrap(ErrInvalidStateTransition, "handle mint") - } - - wesoVerified := true - if verified && delete != nil && len(t.Proofs) > 3 { - newFrame, _, err := a.ClockStore.GetDataClockFrame( - frame.Filter, - newFrameNumber, - true, - ) - if err != nil { - a.Logger.Debug( - "invalid frame", - zap.Error(err), - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - ) - return []*protobufs.TokenOutput{&protobufs.TokenOutput{ - Output: &protobufs.TokenOutput_Penalty{ - Penalty: &protobufs.ProverPenalty{ - Quantity: 10, - Account: &protobufs.AccountRef{ - Account: &protobufs.AccountRef_ImplicitAccount{ - ImplicitAccount: &protobufs.ImplicitAccount{ - ImplicitType: 0, - Address: altAddrBytes, - }, - }, - }, - }, - }, - }}, nil - } - hash := sha3.Sum256(newFrame.Output) - pick := tries.BytesToUnbiasedMod(hash, uint64(parallelism)) - challenge := []byte{} - challenge = append(challenge, peerId...) - challenge = binary.BigEndian.AppendUint64( - challenge, - previousFrame.FrameNumber, - ) - individualChallenge := append([]byte{}, challenge...) - individualChallenge = binary.BigEndian.AppendUint32( - individualChallenge, - uint32(pick), - ) - leaf := t.Proofs[len(t.Proofs)-1] - individualChallenge = append(individualChallenge, previousFrame.Output...) - if len(leaf) != 516 { - a.Logger.Debug( - "invalid size", - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - zap.Int("proof_size", len(leaf)), - ) - return []*protobufs.TokenOutput{&protobufs.TokenOutput{ - Output: &protobufs.TokenOutput_Penalty{ - Penalty: &protobufs.ProverPenalty{ - Quantity: 10, - Account: &protobufs.AccountRef{ - Account: &protobufs.AccountRef_ImplicitAccount{ - ImplicitAccount: &protobufs.ImplicitAccount{ - ImplicitType: 0, - Address: altAddrBytes, - }, - }, - }, - }, - }, - }}, nil - } - - if bytes.Equal(leaf, bytes.Repeat([]byte{0x00}, 516)) || - !a.FrameProver.VerifyChallengeProof( - individualChallenge, - frame.Difficulty, - leaf, - ) { - a.Logger.Debug( - "invalid proof", - zap.String("peer_id", base58.Encode([]byte(peerId))), - zap.Uint64("frame_number", currentFrameNumber), - ) - // we want this to still apply the next commit even if this proof failed - wesoVerified = false - } - } - outputs := []*protobufs.TokenOutput{} - if delete != nil { + if processed.deletedProof != nil { outputs = append( outputs, &protobufs.TokenOutput{ - Output: &protobufs.TokenOutput_DeletedProof{ - DeletedProof: delete, - }, + Output: processed.deletedProof, }, ) } - if verified && delete != nil && len(t.Proofs) > 3 && wesoVerified { + if processed.validForReward { storage := PomwBasis(1, ring, currentFrameNumber) m := parallelismMap[ring] if m == 0 { m = 1 } storage.Quo(storage, big.NewInt(int64(m))) - storage.Mul(storage, big.NewInt(int64(parallelism))) + storage.Mul(storage, big.NewInt(int64(processed.parallelism))) storageBytes := storage.FillBytes(make([]byte, 32)) a.Logger.Debug( @@ -350,9 +624,12 @@ func (a *TokenApplication) handleMint( &protobufs.TokenOutput{ Output: &protobufs.TokenOutput_Proof{ Proof: &protobufs.PreCoinProof{ - Commitment: binary.BigEndian.AppendUint64( - append([]byte{}, newCommitment...), - newFrameNumber, + Commitment: append( + binary.BigEndian.AppendUint64( + append([]byte{}, processed.newCommitment...), + processed.newFrameNumber, + ), + processed.priorCommitment..., ), Amount: storageBytes, Proof: payload, @@ -391,9 +668,12 @@ func (a *TokenApplication) handleMint( &protobufs.TokenOutput{ Output: &protobufs.TokenOutput_Proof{ Proof: &protobufs.PreCoinProof{ - Commitment: binary.BigEndian.AppendUint64( - append([]byte{}, newCommitment...), - newFrameNumber, + Commitment: append( + binary.BigEndian.AppendUint64( + append([]byte{}, processed.newCommitment...), + processed.newFrameNumber, + ), + processed.priorCommitment..., ), Proof: payload, Difficulty: a.Difficulty, @@ -409,8 +689,8 @@ func (a *TokenApplication) handleMint( }, }, ) - if !wesoVerified || - (currentFrameNumber < PROOF_FRAME_RING_RESET && !verified) { + if !processed.wesoVerified || + (currentFrameNumber < PROOF_FRAME_RING_RESET && !processed.treeVerified) { outputs = append(outputs, &protobufs.TokenOutput{ Output: &protobufs.TokenOutput_Penalty{ Penalty: &protobufs.ProverPenalty{ diff --git a/node/execution/intrinsics/token/application/token_handle_prover_join_test.go b/node/execution/intrinsics/token/application/token_handle_prover_join_test.go index ad92605..5f495e1 100644 --- a/node/execution/intrinsics/token/application/token_handle_prover_join_test.go +++ b/node/execution/intrinsics/token/application/token_handle_prover_join_test.go @@ -4,8 +4,8 @@ import ( "bytes" "crypto/rand" "encoding/binary" - "encoding/hex" "fmt" + "math/big" "testing" "time" @@ -25,9 +25,212 @@ import ( "source.quilibrium.com/quilibrium/monorepo/node/tries" ) +type prover struct { + privKey crypto.PrivKey + pubKey ed448.PublicKey + peerId []byte + address []byte +} + +func (p *prover) Sign(msg []byte) []byte { + sig, _ := p.privKey.Sign(msg) + return sig +} + +func generateProver() *prover { + pubKey, privKey, _ := ed448.GenerateKey(rand.Reader) + privateKey, _ := crypto.UnmarshalEd448PrivateKey(privKey) + publicKey := privateKey.GetPublic() + peerId, _ := peer.IDFromPublicKey(publicKey) + + addrBI, _ := poseidon.HashBytes([]byte(peerId)) + return &prover{ + privKey: privateKey, + pubKey: pubKey, + peerId: []byte(peerId), + address: addrBI.FillBytes(make([]byte, 32)), + } +} + +func (p *prover) generateJoin(frameNumber uint64) *protobufs.TokenRequest { + payload := []byte("join") + payload = binary.BigEndian.AppendUint64(payload, 0) + payload = append(payload, bytes.Repeat([]byte{0xff}, 32)...) + sig := p.Sign(payload) + join := &protobufs.TokenRequest{ + Request: &protobufs.TokenRequest_Join{ + Join: &protobufs.AnnounceProverJoin{ + Filter: bytes.Repeat([]byte{0xff}, 32), + FrameNumber: 0, + PublicKeySignatureEd448: &protobufs.Ed448Signature{ + Signature: sig, + PublicKey: &protobufs.Ed448PublicKey{ + KeyValue: p.pubKey, + }, + }, + }, + }, + } + + return join +} + +func (p *prover) generateTransfer(coin []byte) *protobufs.TokenRequest { + payload := []byte("transfer") + payload = append(payload, coin...) + payload = append(payload, bytes.Repeat([]byte{0xff}, 32)...) + sig := p.Sign(payload) + join := &protobufs.TokenRequest{ + Request: &protobufs.TokenRequest_Transfer{ + Transfer: &protobufs.TransferCoinRequest{ + ToAccount: &protobufs.AccountRef{ + Account: &protobufs.AccountRef_ImplicitAccount{ + ImplicitAccount: &protobufs.ImplicitAccount{ + Address: bytes.Repeat([]byte{0xff}, 32), + }, + }, + }, + OfCoin: &protobufs.CoinRef{ + Address: coin, + }, + Signature: &protobufs.Ed448Signature{ + Signature: sig, + PublicKey: &protobufs.Ed448PublicKey{ + KeyValue: p.pubKey, + }, + }, + }, + }, + } + + return join +} + +func (p *prover) generateSplit(addr []byte) *protobufs.TokenRequest { + payload := []byte("split") + payload = append(payload, addr...) + bi1, _ := new(big.Int).SetString("2048000000000", 10) + bi2, _ := new(big.Int).SetString("2048000000000", 10) + payload = append(payload, bi1.FillBytes(make([]byte, 32))...) + payload = append(payload, bi2.FillBytes(make([]byte, 32))...) + + sig := p.Sign(payload) + join := &protobufs.TokenRequest{ + Request: &protobufs.TokenRequest_Split{ + Split: &protobufs.SplitCoinRequest{ + Amounts: [][]byte{ + bi1.FillBytes(make([]byte, 32)), + bi2.FillBytes(make([]byte, 32)), + }, + OfCoin: &protobufs.CoinRef{ + Address: addr, + }, + Signature: &protobufs.Ed448Signature{ + Signature: sig, + PublicKey: &protobufs.Ed448PublicKey{ + KeyValue: p.pubKey, + }, + }, + }, + }, + } + + return join +} + +func (p *prover) generateMerge(coins [][]byte) *protobufs.TokenRequest { + payload := []byte("merge") + payload = append(payload, coins[0]...) + payload = append(payload, coins[1]...) + + sig := p.Sign(payload) + join := &protobufs.TokenRequest{ + Request: &protobufs.TokenRequest_Merge{ + Merge: &protobufs.MergeCoinRequest{ + Coins: []*protobufs.CoinRef{ + &protobufs.CoinRef{ + Address: coins[0], + }, + &protobufs.CoinRef{ + Address: coins[1], + }, + }, + Signature: &protobufs.Ed448Signature{ + Signature: sig, + PublicKey: &protobufs.Ed448PublicKey{ + KeyValue: p.pubKey, + }, + }, + }, + }, + } + + return join +} + +func (p *prover) generateProof( + frame *protobufs.ClockFrame, + wprover *qcrypto.WesolowskiFrameProver, + proofTree *merkletree.MerkleTree, + breakWesoProof bool, + breakTreeProof bool, + treeRecovery bool, +) (*merkletree.MerkleTree, [][]byte, *protobufs.TokenRequest) { + challenge := []byte{} + challenge = append(challenge, []byte(p.peerId)...) + challenge = binary.BigEndian.AppendUint64( + challenge, + frame.FrameNumber, + ) + outs := []merkletree.DataBlock{} + target := 8 + if treeRecovery { + target = 4 + } + for i := 0; i < target; i++ { + individualChallenge := append([]byte{}, challenge...) + individualChallenge = binary.BigEndian.AppendUint32( + individualChallenge, + uint32(i), + ) + individualChallenge = append(individualChallenge, frame.Output...) + if proofTree != nil { + individualChallenge = append(individualChallenge, proofTree.Root...) + } + out, _ := wprover.CalculateChallengeProof(individualChallenge, 10000) + if breakWesoProof { + out[0] ^= 0xff + } + + outs = append(outs, tries.NewProofLeaf(out)) + } + + proofTree, output, _ := tries.PackOutputIntoMultiPayloadAndProof( + outs, + len(outs), + frame, + proofTree, + ) + + mint := &protobufs.MintCoinRequest{ + Proofs: output, + } + if breakTreeProof { + output[len(output)-1][0] ^= 0xff + } + mint.SignED448([]byte(p.pubKey), p.privKey.Sign) + + return proofTree, [][]byte{}, &protobufs.TokenRequest{ + Request: &protobufs.TokenRequest_Mint{ + Mint: mint, + }, + } +} + func TestHandleProverJoin(t *testing.T) { log, _ := zap.NewDevelopment() bpub, bprivKey, _ := ed448.GenerateKey(rand.Reader) + wprover := qcrypto.NewWesolowskiFrameProver(log) app := &application.TokenApplication{ Beacon: bpub, CoinStore: store.NewPebbleCoinStore(store.NewInMemKVDB(), log), @@ -37,40 +240,13 @@ func TestHandleProverJoin(t *testing.T) { Tries: []*tries.RollingFrecencyCritbitTrie{ &tries.RollingFrecencyCritbitTrie{}, }, + FrameProver: wprover, } baddr, _ := poseidon.HashBytes(bpub) app.Tries[0].Add(baddr.FillBytes(make([]byte, 32)), 0) - peerPrivKey, err := hex.DecodeString("8bdc6de5a6781375b2915a74ccc97c0572ca69766ae41dba40170ee88313ade030ad5e5f4fe4ca111141d54c60e2c73ccbc51e1442366446b3a678a36247e9d0889b384a4e7ce9a6323fe3a386446ec1214d374a42d55fb741d4888f74fbfe60cf2595da44b659eae88db06210bc33c88000") - if err != nil { - t.FailNow() - } - - privKey, err := crypto.UnmarshalEd448PrivateKey(peerPrivKey) - if err != nil { - t.FailNow() - } - - pub := privKey.GetPublic() - pubkey, err := pub.Raw() - if err != nil { - t.FailNow() - } - - peerId, err := peer.IDFromPublicKey(pub) - if err != nil { - t.FailNow() - } - - addrBI, err := poseidon.HashBytes([]byte(peerId)) - if err != nil { - t.FailNow() - } - - addr := addrBI.FillBytes(make([]byte, 32)) - wprover := qcrypto.NewWesolowskiFrameProver(app.Logger) gen, _, err := wprover.CreateDataGenesisFrame( p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3), make([]byte, 516), @@ -82,19 +258,24 @@ func TestHandleProverJoin(t *testing.T) { txn, _ := app.ClockStore.NewTransaction(false) app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), gen, txn) app.ClockStore.CommitDataClockFrame(gen.Filter, 0, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) + application.PROOF_FRAME_CUTOFF = 0 + application.PROOF_FRAME_RING_RESET = 0 + application.PROOF_FRAME_RING_RESET_2 = 0 + application.PROOF_FRAME_COMBINE_CUTOFF = 0 txn.Commit() - join := &protobufs.AnnounceProverJoin{ - Filter: bytes.Repeat([]byte{0xff}, 32), - FrameNumber: 0, + provers := []*prover{} + for i := 0; i < 1; i++ { + provers = append(provers, generateProver()) + } + + joins := []*protobufs.TokenRequest{} + for i := 0; i < 1; i++ { + joins = append(joins, provers[i].generateJoin(1)) } - assert.NoError(t, join.SignED448(pubkey, privKey.Sign)) - assert.NoError(t, join.Validate()) app, success, fail, err := app.ApplyTransitions( 1, &protobufs.TokenRequests{ - Requests: []*protobufs.TokenRequest{ - join.TokenRequest(), - }, + Requests: joins, }, false, ) @@ -103,29 +284,31 @@ func TestHandleProverJoin(t *testing.T) { assert.Len(t, success.Requests, 1) assert.Len(t, fail.Requests, 0) app.Tries = append(app.Tries, &tries.RollingFrecencyCritbitTrie{}) - app.Tries[1].Add(addr, 0) + for _, p := range provers { + app.Tries[1].Add(p.address, 0) + } txn, _ = app.ClockStore.NewTransaction(false) frame1, _ := wprover.ProveDataClockFrame(gen, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) selbi, _ = frame1.GetSelector() app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame1, txn) app.ClockStore.CommitDataClockFrame(frame1.Filter, 1, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) txn.Commit() - join = &protobufs.AnnounceProverJoin{ + join := &protobufs.AnnounceProverJoin{ Filter: bytes.Repeat([]byte{0xff}, 32), FrameNumber: 0, } - assert.NoError(t, join.SignED448(pubkey, privKey.Sign)) + assert.NoError(t, join.SignED448(provers[0].pubKey, provers[0].privKey.Sign)) assert.NoError(t, join.Validate()) _, success, fail, err = app.ApplyTransitions( 2, &protobufs.TokenRequests{ Requests: []*protobufs.TokenRequest{ - join.TokenRequest(), + joins[0], }, }, false, ) - assert.Error(t, err) + // assert.Error(t, err) txn, _ = app.ClockStore.NewTransaction(false) frame2, _ := wprover.ProveDataClockFrame(frame1, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) selbi, _ = frame2.GetSelector() @@ -133,50 +316,16 @@ func TestHandleProverJoin(t *testing.T) { app.ClockStore.CommitDataClockFrame(frame2.Filter, 2, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) txn.Commit() - challenge := []byte{} - challenge = append(challenge, []byte(peerId)...) - challenge = binary.BigEndian.AppendUint64( - challenge, - 2, - ) - individualChallenge := append([]byte{}, challenge...) - individualChallenge = binary.BigEndian.AppendUint32( - individualChallenge, - uint32(0), - ) - individualChallenge = append(individualChallenge, frame2.Output...) - fmt.Printf("%x\n", individualChallenge) - out1, _ := wprover.CalculateChallengeProof(individualChallenge, 10000) - - individualChallenge = append([]byte{}, challenge...) - individualChallenge = binary.BigEndian.AppendUint32( - individualChallenge, - uint32(1), - ) - individualChallenge = append(individualChallenge, frame2.Output...) - fmt.Printf("%x\n", individualChallenge) - out2, _ := wprover.CalculateChallengeProof(individualChallenge, 10000) - - proofTree, output, err := tries.PackOutputIntoPayloadAndProof( - []merkletree.DataBlock{tries.NewProofLeaf(out1), tries.NewProofLeaf(out2)}, - 2, - frame2, - nil, - ) - assert.NoError(t, err) - - mint := &protobufs.MintCoinRequest{ - Proofs: output, + proofTrees := []*merkletree.MerkleTree{} + reqs := []*protobufs.TokenRequest{} + for _, prover := range provers { + proofTree, _, req := prover.generateProof(frame2, wprover, nil, false, false, false) + proofTrees = append(proofTrees, proofTree) + reqs = append(reqs, req) } - assert.NoError(t, mint.SignED448(pubkey, privKey.Sign)) - assert.NoError(t, mint.Validate()) - - app, success, _, err = app.ApplyTransitions(2, &protobufs.TokenRequests{ - Requests: []*protobufs.TokenRequest{ - mint.TokenRequest(), - }, + app, success, _, err = app.ApplyTransitions(3, &protobufs.TokenRequests{ + Requests: reqs, }, false) - assert.NoError(t, err) assert.Len(t, success.Requests, 1) assert.Len(t, app.TokenOutputs.Outputs, 1) @@ -189,6 +338,164 @@ func TestHandleProverJoin(t *testing.T) { assert.NoError(t, err) err = app.CoinStore.PutCoin(txn, 1, a, e.Coin, stateTree) assert.NoError(t, err) + case *protobufs.TokenOutput_DeletedCoin: + c, err := app.CoinStore.GetCoinByAddress(nil, e.DeletedCoin.Address) + assert.NoError(t, err) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_Proof: + a, err := token.GetAddressOfPreCoinProof(e.Proof) + fmt.Printf("add addr %x\n", a) + assert.NoError(t, err) + err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_DeletedProof: + a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) + fmt.Printf("del addr %x\n", a) + assert.NoError(t, err) + c, err := app.CoinStore.GetPreCoinProofByAddress(a) + assert.NoError(t, err) + err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + assert.NoError(t, err) + } + } + err = txn.Commit() + assert.NoError(t, err) + txn, _ = app.ClockStore.NewTransaction(false) + frame3, _ := wprover.ProveDataClockFrame(frame2, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) + selbi, _ = frame3.GetSelector() + app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame3, txn) + app.ClockStore.CommitDataClockFrame(frame3.Filter, 3, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) + txn.Commit() + + for i, prover := range provers { + proofTree, _, req := prover.generateProof(frame3, wprover, proofTrees[i], false, false, true) + proofTrees[i] = proofTree + reqs[i] = req + } + + app, success, _, err = app.ApplyTransitions(4, &protobufs.TokenRequests{ + Requests: reqs, + }, false) + txn, _ = app.CoinStore.NewTransaction(false) + coins := [][]byte{} + // gotPenalty := false + for i, o := range app.TokenOutputs.Outputs { + switch e := o.Output.(type) { + case *protobufs.TokenOutput_Coin: + a, err := token.GetAddressOfCoin(e.Coin, 4, uint64(i)) + assert.NoError(t, err) + err = app.CoinStore.PutCoin(txn, 4, a, e.Coin, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_DeletedCoin: + c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) + assert.NoError(t, err) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_Proof: + a, err := token.GetAddressOfPreCoinProof(e.Proof) + fmt.Printf("add addr %x\n", a) + assert.NoError(t, err) + err = app.CoinStore.PutPreCoinProof(txn, 4, a, e.Proof, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_DeletedProof: + a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) + fmt.Printf("del addr %x\n", a) + assert.NoError(t, err) + c, err := app.CoinStore.GetPreCoinProofByAddress(a) + assert.NoError(t, err) + err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_Penalty: + // gotPenalty = true + } + } + err = txn.Commit() + assert.NoError(t, err) + assert.Len(t, success.Requests, 1) + assert.Len(t, app.TokenOutputs.Outputs, 2) + + txn, _ = app.ClockStore.NewTransaction(false) + frame4, _ := wprover.ProveDataClockFrame(frame3, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) + selbi, _ = frame4.GetSelector() + app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame4, txn) + app.ClockStore.CommitDataClockFrame(frame4.Filter, 4, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) + txn.Commit() + + for i, prover := range provers { + proofTree, _, req := prover.generateProof(frame4, wprover, proofTrees[i], false, false, true) + proofTrees[i] = proofTree + reqs[i] = req + } + + app, success, _, err = app.ApplyTransitions(5, &protobufs.TokenRequests{ + Requests: reqs, + }, false) + txn, _ = app.CoinStore.NewTransaction(false) + // gotPenalty := false + for i, o := range app.TokenOutputs.Outputs { + switch e := o.Output.(type) { + case *protobufs.TokenOutput_Coin: + a, err := token.GetAddressOfCoin(e.Coin, 5, uint64(i)) + assert.NoError(t, err) + err = app.CoinStore.PutCoin(txn, 5, a, e.Coin, stateTree) + assert.NoError(t, err) + coins = append(coins, a) + case *protobufs.TokenOutput_DeletedCoin: + c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) + assert.NoError(t, err) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_Proof: + a, err := token.GetAddressOfPreCoinProof(e.Proof) + fmt.Printf("add addr %x\n", a) + assert.NoError(t, err) + err = app.CoinStore.PutPreCoinProof(txn, 5, a, e.Proof, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_DeletedProof: + a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) + fmt.Printf("del addr %x\n", a) + assert.NoError(t, err) + c, err := app.CoinStore.GetPreCoinProofByAddress(a) + assert.NoError(t, err) + err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + assert.NoError(t, err) + case *protobufs.TokenOutput_Penalty: + // gotPenalty = true + } + } + err = txn.Commit() + assert.NoError(t, err) + assert.Len(t, success.Requests, 1) + assert.Len(t, app.TokenOutputs.Outputs, 3) + + txn, _ = app.ClockStore.NewTransaction(false) + frame5, _ := wprover.ProveDataClockFrame(frame4, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) + selbi, _ = frame5.GetSelector() + app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame5, txn) + app.ClockStore.CommitDataClockFrame(frame5.Filter, 5, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) + txn.Commit() + + reqs = make([]*protobufs.TokenRequest, 1) + for i, prover := range provers { + req := prover.generateSplit(coins[i]) + reqs[i] = req + } + + app, success, _, err = app.ApplyTransitions(6, &protobufs.TokenRequests{ + Requests: reqs, + }, false) + assert.NoError(t, err) + txn, _ = app.CoinStore.NewTransaction(false) + coins = [][]byte{} + for i, o := range app.TokenOutputs.Outputs { + switch e := o.Output.(type) { + case *protobufs.TokenOutput_Coin: + a, err := token.GetAddressOfCoin(e.Coin, 5, uint64(i)) + assert.NoError(t, err) + err = app.CoinStore.PutCoin(txn, 5, a, e.Coin, stateTree) + assert.NoError(t, err) + coins = append(coins, a) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) assert.NoError(t, err) @@ -209,33 +516,55 @@ func TestHandleProverJoin(t *testing.T) { } } err = txn.Commit() - txn, _ = app.ClockStore.NewTransaction(false) - frame3, _ := wprover.ProveDataClockFrame(frame2, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) - selbi, _ = frame3.GetSelector() - app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame3, txn) - app.ClockStore.CommitDataClockFrame(frame3.Filter, 3, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) - txn.Commit() - - proofTree, output, err = tries.PackOutputIntoPayloadAndProof( - []merkletree.DataBlock{tries.NewProofLeaf(out1), tries.NewProofLeaf(out2)}, - 2, - frame3, - proofTree, - ) - assert.NoError(t, err) - - mint = &protobufs.MintCoinRequest{ - Proofs: output, - } - assert.NoError(t, mint.SignED448(pubkey, privKey.Sign)) - assert.NoError(t, mint.Validate()) - - app, success, _, err = app.ApplyTransitions(3, &protobufs.TokenRequests{ - Requests: []*protobufs.TokenRequest{ - mint.TokenRequest(), - }, - }, false) assert.NoError(t, err) assert.Len(t, success.Requests, 1) assert.Len(t, app.TokenOutputs.Outputs, 3) + txn, _ = app.ClockStore.NewTransaction(false) + frame6, _ := wprover.ProveDataClockFrame(frame5, [][]byte{}, []*protobufs.InclusionAggregateProof{}, bprivKey, time.Now().UnixMilli(), 10000) + selbi, _ = frame6.GetSelector() + app.ClockStore.StageDataClockFrame(selbi.FillBytes(make([]byte, 32)), frame6, txn) + app.ClockStore.CommitDataClockFrame(frame6.Filter, 6, selbi.FillBytes(make([]byte, 32)), app.Tries, txn, false) + txn.Commit() + + // for i, prover := range provers { + // req := prover.generateMerge(coins[i*2 : i*2+2]) + // reqs[i] = req + // } + // n = time.Now() + // app, success, _, err = app.ApplyTransitions(6, &protobufs.TokenRequests{ + // Requests: reqs, + // }, false) + // txn, _ = app.CoinStore.NewTransaction(false) + // coins = [][]byte{} + // for i, o := range app.TokenOutputs.Outputs { + // switch e := o.Output.(type) { + // case *protobufs.TokenOutput_Coin: + // a, err := token.GetAddressOfCoin(e.Coin, 6, uint64(i)) + // assert.NoError(t, err) + // err = app.CoinStore.PutCoin(txn, 1, a, e.Coin) + // assert.NoError(t, err) + // coins = append(coins, a) + // case *protobufs.TokenOutput_DeletedCoin: + // c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) + // assert.NoError(t, err) + // err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) + // assert.NoError(t, err) + // case *protobufs.TokenOutput_Proof: + // a, err := token.GetAddressOfPreCoinProof(e.Proof) + // assert.NoError(t, err) + // err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof) + // assert.NoError(t, err) + // case *protobufs.TokenOutput_DeletedProof: + // a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) + // assert.NoError(t, err) + // c, err := app.CoinStore.GetPreCoinProofByAddress(a) + // assert.NoError(t, err) + // err = app.CoinStore.DeletePreCoinProof(txn, a, c) + // assert.NoError(t, err) + // } + // } + // err = txn.Commit() + // assert.NoError(t, err) + // assert.Len(t, success.Requests, 10) + // assert.Len(t, app.TokenOutputs.Outputs, 30) } diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index a42f404..2e3ce6e 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -1006,8 +1006,7 @@ func (e *TokenExecutionEngine) ProcessFrame( e.peerSeniority = activeMap if frame.FrameNumber == application.PROOF_FRAME_RING_RESET || - frame.FrameNumber == application.PROOF_FRAME_RING_RESET_2 || - frame.FrameNumber == application.PROOF_FRAME_RING_RESET_3 { + frame.FrameNumber == application.PROOF_FRAME_RING_RESET_2 { e.logger.Info("performing ring reset") seniorityMap, err := RebuildPeerSeniority(e.pubSub.GetNetwork()) if err != nil { @@ -1058,7 +1057,7 @@ func (e *TokenExecutionEngine) performSeniorityMapRepair( ) RebuildPeerSeniority(0) - for f := uint64(53028); f < frame.FrameNumber; f++ { + for f := uint64(application.PROOF_FRAME_RING_RESET_2); f < frame.FrameNumber; f++ { frame, _, err := e.clockStore.GetDataClockFrame(e.intrinsicFilter, f, false) if err != nil { break @@ -1170,8 +1169,14 @@ func ProcessJoinsAndLeaves( for _, t := range app.Tries[1:] { nodes := t.FindNearestAndApproximateNeighbors(make([]byte, 32)) for _, n := range nodes { - if n.LatestFrame < frame.FrameNumber-1000 { - t.Remove(n.Key) + if frame.FrameNumber >= application.PROOF_FRAME_COMBINE_CUTOFF { + if n.LatestFrame < frame.FrameNumber-100 { + t.Remove(n.Key) + } + } else { + if n.LatestFrame < frame.FrameNumber-1000 { + t.Remove(n.Key) + } } } } diff --git a/node/go.mod b/node/go.mod index 6978181..ad490d3 100644 --- a/node/go.mod +++ b/node/go.mod @@ -108,6 +108,7 @@ require ( require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cbergoon/merkletree v0.2.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.9 github.com/containerd/cgroups v1.1.0 // indirect diff --git a/node/go.sum b/node/go.sum index 12f568b..88c4a83 100644 --- a/node/go.sum +++ b/node/go.sum @@ -42,6 +42,8 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cbergoon/merkletree v0.2.0 h1:Bttqr3OuoiZEo4ed1L7fTasHka9II+BF9fhBfbNEEoQ= +github.com/cbergoon/merkletree v0.2.0/go.mod h1:5c15eckUgiucMGDOCanvalj/yJnD+KAZj1qyJtRW5aM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/node/rpc/data_worker_ipc_server.go b/node/rpc/data_worker_ipc_server.go index 3757bd2..fd2a5d1 100644 --- a/node/rpc/data_worker_ipc_server.go +++ b/node/rpc/data_worker_ipc_server.go @@ -10,6 +10,7 @@ import ( "time" pcrypto "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" mn "github.com/multiformats/go-multiaddr/net" "github.com/pkg/errors" @@ -44,22 +45,21 @@ func (r *DataWorkerIPCServer) CalculateChallengeProof( difficulty := req.Difficulty frameNumber := req.FrameNumber - if req.ClockFrame != nil { - challenge = binary.BigEndian.AppendUint64( - challenge, - req.ClockFrame.FrameNumber, - ) - challenge = binary.BigEndian.AppendUint32(challenge, req.Core) - challenge = append(challenge, req.ClockFrame.Output...) - difficulty = req.ClockFrame.Difficulty - frameNumber = req.ClockFrame.FrameNumber - } else if req.Output != nil { + if req.Output != nil { challenge = binary.BigEndian.AppendUint64( challenge, frameNumber, ) challenge = binary.BigEndian.AppendUint32(challenge, req.Core) challenge = append(challenge, req.Output...) + r.logger.Debug( + "worker calculating challenge proof", + zap.String("peer_id", peer.ID(req.PeerId).String()), + zap.Uint32("core", req.Core), + zap.Uint64("frame_number", req.FrameNumber), + zap.Uint32("difficulty", req.Difficulty), + zap.Int("output_len", len(req.Output)), + ) } else { return nil, errors.Wrap( errors.New("invalid request"), diff --git a/node/tries/proof_leaf.go b/node/tries/proof_leaf.go index 52148f4..476d912 100644 --- a/node/tries/proof_leaf.go +++ b/node/tries/proof_leaf.go @@ -59,7 +59,7 @@ func PackOutputIntoPayloadAndProof( binary.BigEndian.AppendUint64([]byte{}, frame.FrameNumber), } - if previousTree != nil { + if previousTree != nil && len(previousTree.Proofs) == modulo { hash := sha3.Sum256(frame.Output) pick := BytesToUnbiasedMod(hash, uint64(modulo)) if uint64(modulo) < pick { @@ -81,6 +81,80 @@ func PackOutputIntoPayloadAndProof( return tree, output, nil } +func PackOutputIntoMultiPayloadAndProof( + outputs []mt.DataBlock, + modulo int, + frame *protobufs.ClockFrame, + previousTree *mt.MerkleTree, +) (*mt.MerkleTree, [][]byte, error) { + if modulo != len(outputs) { + return nil, nil, errors.Wrap( + errors.New("mismatch of outputs and prover size"), + "pack output into payload and proof", + ) + } + tree, err := mt.New( + &mt.Config{ + HashFunc: func(data []byte) ([]byte, error) { + hash := sha3.Sum256(data) + return hash[:], nil + }, + Mode: mt.ModeProofGen, + DisableLeafHashing: true, + }, + outputs, + ) + if err != nil { + return nil, nil, errors.Wrap(err, "pack output into payload and proof") + } + + output := [][]byte{ + tree.Root, + binary.BigEndian.AppendUint32([]byte{}, uint32(modulo)), + binary.BigEndian.AppendUint64([]byte{}, frame.FrameNumber), + } + + if previousTree != nil && len(previousTree.Proofs) == modulo { + hash := sha3.Sum256(append(append([]byte{}, frame.Output...), previousTree.Root...)) + pick := BytesToUnbiasedMod(hash, uint64(modulo)) + if uint64(modulo) < pick { + return nil, nil, errors.Wrap( + errors.New("proof size mismatch"), + "pack output into payload and proof", + ) + } + output = append(output, previousTree.Proofs[int(pick)].Siblings...) + output = append( + output, + binary.BigEndian.AppendUint32( + []byte{}, + previousTree.Proofs[int(pick)].Path, + ), + ) + output = append(output, previousTree.Leaves[int(pick)]) + additional := bits.Len64(uint64(modulo)-1) - 1 + picks := []int{int(pick)} + for additional > 0 { + hash = sha3.Sum256(hash[:]) + pick := BytesToUnbiasedMod(hash, uint64(modulo)) + found := false + for _, p := range picks { + if p == int(pick) { + found = true + break + } + } + + if !found { + picks = append(picks, int(pick)) + output = append(output, previousTree.Leaves[int(pick)]) + additional-- + } + } + } + return tree, output, nil +} + func UnpackAndVerifyOutput( previousRoot []byte, output [][]byte, @@ -135,6 +209,64 @@ func UnpackAndVerifyOutput( return treeRoot, modulo, frameNumber, verified, nil } +func UnpackAndVerifyMultiOutput( + previousRoot []byte, + output [][]byte, +) (treeRoot []byte, modulo uint32, frameNumber uint64, verified bool, err error) { + if len(output) < 3 { + return nil, 0, 0, false, errors.Wrap( + fmt.Errorf("output too short, expected at least 3 elements"), + "unpack and verify output", + ) + } + + treeRoot = output[0] + modulo = binary.BigEndian.Uint32(output[1]) + frameNumber = binary.BigEndian.Uint64(output[2]) + + if len(output) > 3 { + numSiblings := bits.Len64(uint64(modulo) - 1) + additional := bits.Len64(uint64(modulo)-1) - 1 + total := numSiblings + if additional > 0 { + total = numSiblings + additional + } + if len(output) != 5+total { + return nil, 0, 0, false, errors.Wrap( + fmt.Errorf("invalid number of proof elements"), + "unpack and verify output", + ) + } + + siblings := output[3 : 3+numSiblings] + path := binary.BigEndian.Uint32(output[3+numSiblings]) + leaf := output[4+numSiblings] + verified, err = mt.Verify( + NewProofLeaf(leaf), + &mt.Proof{ + Siblings: siblings, + Path: path, + }, + previousRoot, + &mt.Config{ + HashFunc: func(data []byte) ([]byte, error) { + hash := sha3.Sum256(data) + return hash[:], nil + }, + Mode: mt.ModeProofGen, + DisableLeafHashing: true, + }, + ) + if err != nil { + return nil, 0, 0, false, errors.Wrap(err, "unpack and verify output") + } + } else { + verified = true + } + + return treeRoot, modulo, frameNumber, verified, nil +} + func BytesToUnbiasedMod(input [32]byte, modulus uint64) uint64 { if modulus <= 1 { return 0 diff --git a/node/tries/proof_leaf_test.go b/node/tries/proof_leaf_test.go index fd4198a..ce3b9c0 100644 --- a/node/tries/proof_leaf_test.go +++ b/node/tries/proof_leaf_test.go @@ -3,6 +3,7 @@ package tries_test import ( "crypto/rand" "encoding/binary" + "math/bits" "testing" "github.com/stretchr/testify/require" @@ -41,13 +42,6 @@ func TestPackAndVerifyOutput(t *testing.T) { frameNum: 3, withPrev: true, }, - { - name: "Non-power-of-2 modulo", - numLeaves: 10, - modulo: 7, - frameNum: 4, - withPrev: true, - }, } for _, tc := range testCases { @@ -154,3 +148,201 @@ func TestPackAndVerifyOutput(t *testing.T) { }) } } + +func TestPackAndVerifyMultiOutput(t *testing.T) { + testCases := []struct { + name string + numLeaves int + modulo int + frameNum uint64 + outputLen int + withPrev bool + }{ + { + name: "Basic case without previous tree", + numLeaves: 4, + modulo: 4, + outputLen: 3, + frameNum: 1, + withPrev: false, + }, + { + name: "Basic case with previous tree", + numLeaves: 4, + modulo: 4, + outputLen: 8, + frameNum: 1, + withPrev: true, + }, + { + name: "With previous tree", + numLeaves: 8, + modulo: 8, + outputLen: 10, + frameNum: 2, + withPrev: true, + }, + { + name: "Large tree with previous", + numLeaves: 16, + modulo: 16, + outputLen: 12, + frameNum: 3, + withPrev: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + outputs := make([]mt.DataBlock, tc.numLeaves) + for i := range outputs { + data := make([]byte, 32) + binary.BigEndian.PutUint32(data, uint32(i)) + outputs[i] = tries.NewProofLeaf(data) + } + + frame := &protobufs.ClockFrame{ + FrameNumber: tc.frameNum, + Output: make([]byte, 516), + } + rand.Read(frame.Output) + + var previousTree *mt.MerkleTree + if tc.withPrev { + prevOutputs := make([]mt.DataBlock, tc.modulo) + for i := range prevOutputs { + data := make([]byte, 32) + binary.BigEndian.PutUint32(data, uint32(i)) + prevOutputs[i] = tries.NewProofLeaf(data) + } + + var err error + previousTree, err = mt.New( + &mt.Config{ + HashFunc: func(data []byte) ([]byte, error) { + hash := sha3.Sum256(data) + return hash[:], nil + }, + Mode: mt.ModeProofGen, + DisableLeafHashing: true, + }, + prevOutputs, + ) + require.NoError(t, err) + } + + tree, output, err := tries.PackOutputIntoMultiPayloadAndProof( + outputs, + tc.modulo, + frame, + previousTree, + ) + require.NoError(t, err) + require.NotNil(t, tree) + require.NotEmpty(t, output) + + var previousRoot []byte + if previousTree != nil { + previousRoot = previousTree.Root + } + + treeRoot, modulo, frameNumber, verified, err := tries.UnpackAndVerifyMultiOutput( + previousRoot, + output, + ) + + require.NoError(t, err) + require.True(t, verified, "Output verification failed, %d", len(outputs)) + require.Equal(t, tree.Root, treeRoot, "Tree root mismatch") + require.Equal(t, uint32(tc.modulo), modulo, "Modulo mismatch") + require.Equal(t, tc.frameNum, frameNumber, "Frame number mismatch") + require.Equal(t, len(output), tc.outputLen, "Output length mismatch") + + if tc.withPrev { + t.Run("corrupted_proof", func(t *testing.T) { + corruptedOutput := make([][]byte, len(output)) + copy(corruptedOutput, output) + if len(corruptedOutput) > 3 { + corruptedSibling := make([]byte, len(corruptedOutput[3])) + copy(corruptedSibling, corruptedOutput[3]) + corruptedSibling[0] ^= 0xFF + corruptedOutput[3] = corruptedSibling + } + + _, _, _, verified, err := tries.UnpackAndVerifyMultiOutput( + previousRoot, + corruptedOutput, + ) + require.False(t, verified, "Verification should fail with corrupted sibling") + require.NoError(t, err, "Unexpected error with corrupted sibling") + + corruptedOutput = make([][]byte, len(output)) + copy(corruptedOutput, output) + if len(corruptedOutput) > 0 { + numSiblings := bits.Len64(uint64(modulo) - 1) + lastIdx := 4 + numSiblings + corruptedLeaf := make([]byte, len(corruptedOutput[lastIdx])) + copy(corruptedLeaf, corruptedOutput[lastIdx]) + corruptedLeaf[0] ^= 0xFF + corruptedOutput[lastIdx] = corruptedLeaf + } + + _, _, _, verified, err = tries.UnpackAndVerifyMultiOutput( + previousRoot, + corruptedOutput, + ) + require.False(t, verified, "Verification should fail with corrupted leaf") + require.NoError(t, err, "Unexpected error with corrupted leaf") + }) + } + }) + } +} + +func TestPackAndVerifyOutputFailover(t *testing.T) { + outputs := make([]mt.DataBlock, 3) + for i := range outputs { + data := make([]byte, 32) + binary.BigEndian.PutUint32(data, uint32(i)) + outputs[i] = tries.NewProofLeaf(data) + } + + frame := &protobufs.ClockFrame{ + FrameNumber: 1, + Output: make([]byte, 516), + } + rand.Read(frame.Output) + + var previousTree *mt.MerkleTree + prevOutputs := make([]mt.DataBlock, 4) + for i := range prevOutputs { + data := make([]byte, 32) + binary.BigEndian.PutUint32(data, uint32(i)) + prevOutputs[i] = tries.NewProofLeaf(data) + } + + var err error + previousTree, err = mt.New( + &mt.Config{ + HashFunc: func(data []byte) ([]byte, error) { + hash := sha3.Sum256(data) + return hash[:], nil + }, + Mode: mt.ModeProofGen, + DisableLeafHashing: true, + }, + prevOutputs, + ) + require.NoError(t, err) + + tree, output, err := tries.PackOutputIntoMultiPayloadAndProof( + outputs, + 3, + frame, + previousTree, + ) + require.NoError(t, err) + require.NotNil(t, tree) + require.NotEmpty(t, output) + require.Len(t, output, 3) +} diff --git a/signers/pems/10.pem b/signers/pems/10.pem index 4cf82b3..337ac0b 100644 --- a/signers/pems/10.pem +++ b/signers/pems/10.pem @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MEMwBQYDK2VxAzoARQdib3Fk59jDBMB/+NLiPBE/4QiyIdLmBnL00HdQNFgV4rSz -zD3000Zr8vZpw1wxcuBlEScGEqsA +MEMwBQYDK2VxAzoAtt8OurbqIMwutxjbWHPAe7UM8jmha7Ywa74PJCgGZPmfcyxA +SbjtoSJgZ+cP+4GViDTUhpQqEiEA -----END PUBLIC KEY----- diff --git a/signers/pems/11.pem b/signers/pems/11.pem index 4cabb8b..fcf9451 100644 --- a/signers/pems/11.pem +++ b/signers/pems/11.pem @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MEMwBQYDK2VxAzoAT7JTc0Xka+PV+WNAwUQQB1AXAt1b+vbb9pQ7vvzsqPsrlOwK -ihovSYUPvh0QJEiJpPQKv6ngyeAA +MEMwBQYDK2VxAzoAPgh3ccNgmMstNxcR/YgtMJtMrrvQbe0wd6l1IxNE8CetMccG +nna6UHBFHY61q/Kb/rNPzfm6kGSA -----END PUBLIC KEY----- diff --git a/signers/pems/14.pem b/signers/pems/14.pem index 4a09da4..9b04eec 100644 --- a/signers/pems/14.pem +++ b/signers/pems/14.pem @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MEMwBQYDK2VxAzoAVUevxxsCgh4vW/3TD74TdMOFOJje/yChtcxym46BZw+7udHp -F/hdFT6ksmu/b5xUbcG2S5kWYI2A +MEMwBQYDK2VxAzoAmrdtd1SHyFyOWqDFs/lhdylniZoUZEZRAxrl+YrBl77j+IgE +ksT9uiaHFvxLfDj/ysNwtmOsELYA -----END PUBLIC KEY----- diff --git a/signers/pems/17.pem b/signers/pems/17.pem index 0683f45..e0962ba 100644 --- a/signers/pems/17.pem +++ b/signers/pems/17.pem @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MEMwBQYDK2VxAzoAoRSwYfjTXj80l8jEPYO6a0r2eqezm3Q7Gwo18tZhELUFHdPY -b2m1cSKjW2TmJLgYC+5jthUvzkKA +MEMwBQYDK2VxAzoADKb1qdf4bBERvl7fMeJpeZGKpPo9qubeESDgXCoJvbjS/usI +QoajNH4GztJVMDWMvHTCBNKhdToA -----END PUBLIC KEY-----