diff --git a/client/go.mod b/client/go.mod index 7cc1261..2e042d9 100644 --- a/client/go.mod +++ b/client/go.mod @@ -24,6 +24,8 @@ replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go- replace source.quilibrium.com/quilibrium/monorepo/node => ../node +replace github.com/cockroachdb/pebble => ../pebble + require ( github.com/iden3/go-iden3-crypto v0.0.16 github.com/multiformats/go-multiaddr v0.12.4 @@ -42,10 +44,9 @@ require ( github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401 // indirect github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/errors v1.11.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v1.1.4 // indirect + github.com/cockroachdb/pebble v0.0.0-20231210175920-b4d301aeb46a // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/gnark-crypto v0.5.3 // indirect @@ -57,7 +58,7 @@ require ( github.com/elastic/gosigar v0.14.2 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect diff --git a/client/go.sum b/client/go.sum index 665d42d..3e21f86 100644 --- a/client/go.sum +++ b/client/go.sum @@ -50,14 +50,12 @@ github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= -github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8= -github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -106,8 +104,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index 92e4ef2..b98bd0f 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -18,22 +18,11 @@ func init() { } const ( - NodesPerBranch = 64 - BranchBits = 6 // log2(64) - BranchMask = NodesPerBranch - 1 + BranchNodes = 64 + BranchBits = 6 // log2(64) + BranchMask = BranchNodes - 1 ) -type VectorCommitmentTree interface { - Commit(recalculate bool) []byte - Delete(key []byte) error - Get(key []byte) ([]byte, error) - GetMetadata() (leafCount int, longestBranch int) - GetSize() *big.Int - Insert(key []byte, value []byte, hashTarget []byte, size *big.Int) error - Prove(key []byte) [][]byte - Verify(key []byte, proofs [][]byte) bool -} - type VectorCommitmentNode interface { Commit(recalculate bool) []byte GetSize() *big.Int @@ -49,7 +38,7 @@ type VectorCommitmentLeafNode struct { type VectorCommitmentBranchNode struct { Prefix []int - Children [NodesPerBranch]VectorCommitmentNode + Children [BranchNodes]VectorCommitmentNode Commitment []byte Size *big.Int LeafCount int @@ -186,12 +175,12 @@ func (n *VectorCommitmentBranchNode) Prove(index int) []byte { return rbls48581.ProveRaw(data, uint64(index), 64) } -type RawVectorCommitmentTree struct { +type VectorCommitmentTree struct { Root VectorCommitmentNode } -// GetNextNibble returns the next BranchBits bits from the key starting at pos -func GetNextNibble(key []byte, pos int) int { +// getNextNibble returns the next BranchBits bits from the key starting at pos +func getNextNibble(key []byte, pos int) int { startByte := pos / 8 if startByte >= len(key) { return 0 @@ -219,13 +208,13 @@ func GetNextNibble(key []byte, pos int) int { return result & BranchMask } -func GetNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) { +func getNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) { var nibbles []int depth := startDepth for { - n1 := GetNextNibble(key1, depth) - n2 := GetNextNibble(key2, depth) + n1 := getNextNibble(key1, depth) + n2 := getNextNibble(key2, depth) if n1 != n2 { return nibbles, depth } @@ -235,7 +224,7 @@ func GetNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) { } // Insert adds or updates a key-value pair in the tree -func (t *RawVectorCommitmentTree) Insert( +func (t *VectorCommitmentTree) Insert( key, value, hashTarget []byte, size *big.Int, ) error { @@ -264,7 +253,7 @@ func (t *RawVectorCommitmentTree) Insert( } // Get common prefix nibbles and divergence point - sharedNibbles, divergeDepth := GetNibblesUntilDiverge(n.Key, key, depth) + sharedNibbles, divergeDepth := getNibblesUntilDiverge(n.Key, key, depth) // Create single branch node with shared prefix branch := &VectorCommitmentBranchNode{ @@ -275,8 +264,8 @@ func (t *RawVectorCommitmentTree) Insert( } // Add both leaves at their final positions - finalOldNibble := GetNextNibble(n.Key, divergeDepth) - finalNewNibble := GetNextNibble(key, divergeDepth) + finalOldNibble := getNextNibble(n.Key, divergeDepth) + finalNewNibble := getNextNibble(key, divergeDepth) branch.Children[finalOldNibble] = n branch.Children[finalNewNibble] = &VectorCommitmentLeafNode{ Key: key, @@ -291,7 +280,7 @@ func (t *RawVectorCommitmentTree) Insert( if len(n.Prefix) > 0 { // Check if the new key matches the prefix for i, expectedNibble := range n.Prefix { - actualNibble := GetNextNibble(key, depth+i*BranchBits) + actualNibble := getNextNibble(key, depth+i*BranchBits) if actualNibble != expectedNibble { // Create new branch with shared prefix subset newBranch := &VectorCommitmentBranchNode{ @@ -314,7 +303,7 @@ func (t *RawVectorCommitmentTree) Insert( } // Key matches prefix, continue with final nibble - finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits) + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) inserted := insert( n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits, @@ -334,7 +323,7 @@ func (t *RawVectorCommitmentTree) Insert( return n } else { // Simple branch without prefix - nibble := GetNextNibble(key, depth) + nibble := getNextNibble(key, depth) inserted := insert(n.Children[nibble], depth+BranchBits) n.Children[nibble] = inserted n.Commitment = nil @@ -359,7 +348,7 @@ func (t *RawVectorCommitmentTree) Insert( return nil } -func (t *RawVectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool { +func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool { if len(key) == 0 { return false } @@ -384,13 +373,13 @@ func (t *RawVectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool { case *VectorCommitmentBranchNode: // Check prefix match for i, expectedNibble := range n.Prefix { - if GetNextNibble(key, depth+i*BranchBits) != expectedNibble { + if getNextNibble(key, depth+i*BranchBits) != expectedNibble { return false } } // Get final nibble after prefix - finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits) + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) if !n.Verify(finalNibble, proofs[0]) { return false @@ -405,7 +394,7 @@ func (t *RawVectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool { return verify(t.Root, proofs, 0) } -func (t *RawVectorCommitmentTree) Prove(key []byte) [][]byte { +func (t *VectorCommitmentTree) Prove(key []byte) [][]byte { if len(key) == 0 { return nil } @@ -426,13 +415,13 @@ func (t *RawVectorCommitmentTree) Prove(key []byte) [][]byte { case *VectorCommitmentBranchNode: // Check prefix match for i, expectedNibble := range n.Prefix { - if GetNextNibble(key, depth+i*BranchBits) != expectedNibble { + if getNextNibble(key, depth+i*BranchBits) != expectedNibble { return nil } } // Get final nibble after prefix - finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits) + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) proofs := [][]byte{n.Prove(finalNibble)} @@ -446,7 +435,7 @@ func (t *RawVectorCommitmentTree) Prove(key []byte) [][]byte { } // Get retrieves a value from the tree by key -func (t *RawVectorCommitmentTree) Get(key []byte) ([]byte, error) { +func (t *VectorCommitmentTree) Get(key []byte) ([]byte, error) { if len(key) == 0 { return nil, errors.New("empty key not allowed") } @@ -467,12 +456,12 @@ func (t *RawVectorCommitmentTree) Get(key []byte) ([]byte, error) { case *VectorCommitmentBranchNode: // Check prefix match for i, expectedNibble := range n.Prefix { - if GetNextNibble(key, depth+i*BranchBits) != expectedNibble { + if getNextNibble(key, depth+i*BranchBits) != expectedNibble { return nil } } // Get final nibble after prefix - finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits) + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) return get(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) } @@ -487,7 +476,7 @@ func (t *RawVectorCommitmentTree) Get(key []byte) ([]byte, error) { } // Delete removes a key-value pair from the tree -func (t *RawVectorCommitmentTree) Delete(key []byte) error { +func (t *VectorCommitmentTree) Delete(key []byte) error { if len(key) == 0 { return errors.New("empty key not allowed") } @@ -508,13 +497,13 @@ func (t *RawVectorCommitmentTree) Delete(key []byte) error { case *VectorCommitmentBranchNode: for i, expectedNibble := range n.Prefix { - currentNibble := GetNextNibble(key, depth+i*BranchBits) + currentNibble := getNextNibble(key, depth+i*BranchBits) if currentNibble != expectedNibble { return big.NewInt(0), n } } - finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits) + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) var size *big.Int size, n.Children[finalNibble] = remove(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) @@ -579,10 +568,7 @@ func (t *RawVectorCommitmentTree) Delete(key []byte) error { return nil } -func (t *RawVectorCommitmentTree) GetMetadata() ( - leafCount int, - longestBranch int, -) { +func (t *VectorCommitmentTree) GetMetadata() (leafCount int, longestBranch int) { switch root := t.Root.(type) { case nil: return 0, 0 @@ -595,14 +581,14 @@ func (t *RawVectorCommitmentTree) GetMetadata() ( } // Commit returns the root of the tree -func (t *RawVectorCommitmentTree) Commit(recalculate bool) []byte { +func (t *VectorCommitmentTree) Commit(recalculate bool) []byte { if t.Root == nil { return make([]byte, 64) } return t.Root.Commit(recalculate) } -func (t *RawVectorCommitmentTree) GetSize() *big.Int { +func (t *VectorCommitmentTree) GetSize() *big.Int { return t.Root.GetSize() } diff --git a/node/crypto/proof_tree_test.go b/node/crypto/proof_tree_test.go index 31bd448..7f12769 100644 --- a/node/crypto/proof_tree_test.go +++ b/node/crypto/proof_tree_test.go @@ -11,7 +11,7 @@ import ( ) func BenchmarkVectorCommitmentTreeInsert(b *testing.B) { - tree := &RawVectorCommitmentTree{} + tree := &VectorCommitmentTree{} addresses := [][]byte{} for i := range b.N { @@ -26,7 +26,7 @@ func BenchmarkVectorCommitmentTreeInsert(b *testing.B) { } func BenchmarkVectorCommitmentTreeCommit(b *testing.B) { - tree := &RawVectorCommitmentTree{} + tree := &VectorCommitmentTree{} addresses := [][]byte{} for i := range b.N { @@ -42,7 +42,7 @@ func BenchmarkVectorCommitmentTreeCommit(b *testing.B) { } func BenchmarkVectorCommitmentTreeProve(b *testing.B) { - tree := &RawVectorCommitmentTree{} + tree := &VectorCommitmentTree{} addresses := [][]byte{} for i := range b.N { @@ -58,7 +58,7 @@ func BenchmarkVectorCommitmentTreeProve(b *testing.B) { } func BenchmarkVectorCommitmentTreeVerify(b *testing.B) { - tree := &RawVectorCommitmentTree{} + tree := &VectorCommitmentTree{} addresses := [][]byte{} for i := range b.N { @@ -78,7 +78,7 @@ func BenchmarkVectorCommitmentTreeVerify(b *testing.B) { func TestVectorCommitmentTrees(t *testing.T) { bls48581.Init() - tree := &RawVectorCommitmentTree{} + tree := &VectorCommitmentTree{} // Test single insert err := tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1)) @@ -106,7 +106,7 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Error("Expected error for empty key, got none") } - tree = &RawVectorCommitmentTree{} + tree = &VectorCommitmentTree{} // Test get on empty tree _, err = tree.Get([]byte("nonexistent")) @@ -130,7 +130,7 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Error("Expected error for empty key, got none") } - tree = &RawVectorCommitmentTree{} + tree = &VectorCommitmentTree{} // Test delete on empty tree err = tree.Delete([]byte("nonexistent")) @@ -157,7 +157,7 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Error("Expected error for empty key, got none") } - tree = &RawVectorCommitmentTree{} + tree = &VectorCommitmentTree{} // Insert keys that share common prefix keys := []string{ @@ -212,7 +212,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } } - tree = &RawVectorCommitmentTree{} + tree = &VectorCommitmentTree{} // Empty tree should be empty emptyRoot := tree.Root @@ -244,8 +244,8 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Error("Root hash should match empty tree after deleting all entries") } - tree = &RawVectorCommitmentTree{} - cmptree := &RawVectorCommitmentTree{} + tree = &VectorCommitmentTree{} + cmptree := &VectorCommitmentTree{} addresses := [][]byte{} @@ -332,7 +332,7 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Errorf("invalid tree size: %s", tree.GetSize().String()) } - cmptree = &RawVectorCommitmentTree{} + cmptree = &VectorCommitmentTree{} for i := 0; i < 10000; i++ { cmptree.Insert(kept[i], kept[i], nil, big.NewInt(1)) diff --git a/node/crypto/tree_compare.go b/node/crypto/tree_compare.go index d15c8ed..a52c744 100644 --- a/node/crypto/tree_compare.go +++ b/node/crypto/tree_compare.go @@ -6,7 +6,7 @@ import ( ) // CompareTreesAtHeight compares two vector commitment trees at each level -func CompareTreesAtHeight(tree1, tree2 *RawVectorCommitmentTree) [][]ComparisonResult { +func CompareTreesAtHeight(tree1, tree2 *VectorCommitmentTree) [][]ComparisonResult { if tree1 == nil || tree2 == nil { return nil } @@ -104,7 +104,7 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren // If we're still below target height after prefix, traverse children if nextHeight < targetHeight { - for i := 0; i < NodesPerBranch; i++ { + for i := 0; i < BranchNodes; i++ { childResults := compareLevelCommits(n1.Children[i], n2.Children[i], targetHeight, nextHeight+1) results = append(results, childResults...) } @@ -115,7 +115,7 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren } // TraverseAndCompare provides a channel-based iterator for comparing trees -func TraverseAndCompare(tree1, tree2 *RawVectorCommitmentTree) chan ComparisonResult { +func TraverseAndCompare(tree1, tree2 *VectorCommitmentTree) chan ComparisonResult { resultChan := make(chan ComparisonResult) go func() { @@ -150,7 +150,7 @@ type LeafDifference struct { } // CompareLeaves returns all leaves that differ between the two trees -func CompareLeaves(tree1, tree2 *RawVectorCommitmentTree) []LeafDifference { +func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { // Get all leaves from both trees leaves1 := GetAllLeaves(tree1.Root) leaves2 := GetAllLeaves(tree2.Root) @@ -231,8 +231,8 @@ func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { func ExampleComparison() { // Create and populate two trees - tree1 := &RawVectorCommitmentTree{} - tree2 := &RawVectorCommitmentTree{} + tree1 := &VectorCommitmentTree{} + tree2 := &VectorCommitmentTree{} // Compare trees using channel-based iterator for result := range TraverseAndCompare(tree1, tree2) { diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 97c7ed9..a045858 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -148,18 +148,7 @@ func NewTokenExecutionEngine( var inclusionProof *qcrypto.InclusionAggregateProof var proverKeys [][]byte var peerSeniority map[string]uint64 - hg := hypergraph.NewHypergraph( - func( - shardKey hypergraph.ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - ) qcrypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - hypergraphStore, - shardKey, - phaseSet, - ) - }, - ) + hg := hypergraph.NewHypergraph() mpcithVerEnc := qcrypto.NewMPCitHVerifiableEncryptor( runtime.NumCPU(), ) @@ -495,7 +484,7 @@ func (e *TokenExecutionEngine) addBatchToHypergraph(batchKey [][]byte, batchValu var wg sync.WaitGroup throttle := make(chan struct{}, runtime.NumCPU()) batchCompressed := make([]hypergraph.Vertex, len(batchKey)) - batchTrees := make([]*qcrypto.RawVectorCommitmentTree, len(batchKey)) + batchTrees := make([]*qcrypto.VectorCommitmentTree, len(batchKey)) txn, err := e.hypergraphStore.NewTransaction(false) if err != nil { panic(err) @@ -650,18 +639,7 @@ func (e *TokenExecutionEngine) hyperSync() { func (e *TokenExecutionEngine) rebuildHypergraph() { e.logger.Info("rebuilding hypergraph") - e.hypergraph = hypergraph.NewHypergraph( - func( - shardKey hypergraph.ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - ) qcrypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - e.hypergraphStore, - shardKey, - phaseSet, - ) - }, - ) + e.hypergraph = hypergraph.NewHypergraph() if e.engineConfig.RebuildStart == "" { e.engineConfig.RebuildStart = "0000000000000000000000000000000000000000000000000000000000000000" } diff --git a/node/go.mod b/node/go.mod index 1bade8f..5c5e122 100644 --- a/node/go.mod +++ b/node/go.mod @@ -23,8 +23,10 @@ replace github.com/libp2p/go-libp2p-kad-dht => ../go-libp2p-kad-dht replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go-libp2p-blossomsub +replace github.com/cockroachdb/pebble => ../pebble + require ( - github.com/cockroachdb/pebble v1.1.4 + github.com/cockroachdb/pebble v0.0.0-20231210175920-b4d301aeb46a github.com/deiu/rdf2go v0.0.0-20240619132609-81222e324bb9 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/klauspost/reedsolomon v1.12.4 @@ -43,8 +45,8 @@ require ( require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/deiu/gon3 v0.0.0-20230411081920-f0f8f879f597 // indirect + github.com/google/subcommands v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 // indirect @@ -78,14 +80,14 @@ require ( github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/charmbracelet/bubbletea v0.24.2 github.com/charmbracelet/lipgloss v0.9.1 - github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/gnark-crypto v0.5.3 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 @@ -110,6 +112,7 @@ require ( require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cbergoon/merkletree v0.2.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.9 github.com/containerd/cgroups v1.1.0 // indirect diff --git a/node/go.sum b/node/go.sum index 3c18536..acefbe5 100644 --- a/node/go.sum +++ b/node/go.sum @@ -42,6 +42,8 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cbergoon/merkletree v0.2.0 h1:Bttqr3OuoiZEo4ed1L7fTasHka9II+BF9fhBfbNEEoQ= +github.com/cbergoon/merkletree v0.2.0/go.mod h1:5c15eckUgiucMGDOCanvalj/yJnD+KAZj1qyJtRW5aM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -56,14 +58,12 @@ github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= -github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8= -github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= +github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -117,8 +117,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -183,6 +183,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/node/hypergraph/application/hypergraph.go b/node/hypergraph/application/hypergraph.go index c7c1d0c..35932c1 100644 --- a/node/hypergraph/application/hypergraph.go +++ b/node/hypergraph/application/hypergraph.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/p2p" - "source.quilibrium.com/quilibrium/monorepo/node/protobufs" ) type AtomType string @@ -73,7 +72,7 @@ type hyperedge struct { appAddress [32]byte dataAddress [32]byte extrinsics map[[64]byte]Atom - extTree *crypto.RawVectorCommitmentTree + extTree *crypto.VectorCommitmentTree } var _ Vertex = (*vertex)(nil) @@ -90,8 +89,8 @@ type Atom interface { Commit() []byte } -func EncryptedToVertexTree(encrypted []Encrypted) *crypto.RawVectorCommitmentTree { - dataTree := &crypto.RawVectorCommitmentTree{} +func EncryptedToVertexTree(encrypted []Encrypted) *crypto.VectorCommitmentTree { + dataTree := &crypto.VectorCommitmentTree{} for _, d := range encrypted { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -119,7 +118,7 @@ func AtomFromBytes(data []byte) Atom { size: new(big.Int).SetBytes(data[len(data)-32:]), } } else { - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer b.Write(data[65:]) dec := gob.NewDecoder(&b) @@ -163,7 +162,7 @@ func NewHyperedge( appAddress: appAddress, dataAddress: dataAddress, extrinsics: make(map[[64]byte]Atom), - extTree: &crypto.RawVectorCommitmentTree{}, + extTree: &crypto.VectorCommitmentTree{}, } } @@ -331,38 +330,48 @@ type IdSet struct { dirty bool atomType AtomType atoms map[[64]byte]Atom - tree crypto.VectorCommitmentTree + tree *crypto.VectorCommitmentTree } -func NewIdSet( - shardKey ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - tree crypto.VectorCommitmentTree, -) *IdSet { - atomType := VertexAtomType - switch phaseSet { - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS: - fallthrough - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES: - atomType = VertexAtomType - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS: - fallthrough - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES: - atomType = HyperedgeAtomType - } - +func NewIdSet(atomType AtomType) *IdSet { return &IdSet{ dirty: false, atomType: atomType, atoms: make(map[[64]byte]Atom), - tree: tree, + tree: &crypto.VectorCommitmentTree{}, } } +func (set *IdSet) FromBytes(treeData []byte) error { + set.tree = &crypto.VectorCommitmentTree{} + var b bytes.Buffer + b.Write(treeData) + dec := gob.NewDecoder(&b) + if err := dec.Decode(set.tree); err != nil { + return errors.Wrap(err, "load set") + } + + for _, leaf := range crypto.GetAllLeaves(set.tree.Root) { + set.atoms[[64]byte(leaf.Key)] = AtomFromBytes(leaf.Value) + } + + return nil +} + func (set *IdSet) IsDirty() bool { return set.dirty } +func (set *IdSet) ToBytes() []byte { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(set.tree); err != nil { + return nil + } + + return buf.Bytes() +} + func (set *IdSet) Add(atom Atom) error { if atom.GetAtomType() != set.atomType { return ErrInvalidAtomType @@ -403,31 +412,21 @@ func (set *IdSet) Has(key [64]byte) bool { return ok } -func (set *IdSet) GetTree() crypto.VectorCommitmentTree { +func (set *IdSet) GetTree() *crypto.VectorCommitmentTree { return set.tree } type Hypergraph struct { - size *big.Int - treeConstructor func( - shardKey ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - ) crypto.VectorCommitmentTree + size *big.Int vertexAdds map[ShardKey]*IdSet vertexRemoves map[ShardKey]*IdSet hyperedgeAdds map[ShardKey]*IdSet hyperedgeRemoves map[ShardKey]*IdSet } -func NewHypergraph( - treeConstructor func( - shardKey ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - ) crypto.VectorCommitmentTree, -) *Hypergraph { +func NewHypergraph() *Hypergraph { return &Hypergraph{ size: big.NewInt(0), - treeConstructor: treeConstructor, vertexAdds: make(map[ShardKey]*IdSet), vertexRemoves: make(map[ShardKey]*IdSet), hyperedgeAdds: make(map[ShardKey]*IdSet), @@ -468,26 +467,36 @@ func (hg *Hypergraph) Commit() [][]byte { return commits } -func (hg *Hypergraph) SetIdSet( +func (hg *Hypergraph) ImportFromBytes( + atomType AtomType, + phaseType PhaseType, shardKey ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - tree crypto.VectorCommitmentTree, + data []byte, ) error { - set := NewIdSet(shardKey, phaseSet, tree) + set := NewIdSet(atomType) + if err := set.FromBytes(data); err != nil { + return errors.Wrap(err, "import from bytes") + } - switch phaseSet { - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS: - hg.size.Add(hg.size, set.GetSize()) - hg.vertexAdds[shardKey] = set - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES: - hg.size.Sub(hg.size, set.GetSize()) - hg.vertexRemoves[shardKey] = set - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS: - hg.size.Add(hg.size, set.GetSize()) - hg.hyperedgeAdds[shardKey] = set - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES: - hg.size.Sub(hg.size, set.GetSize()) - hg.hyperedgeRemoves[shardKey] = set + switch atomType { + case VertexAtomType: + switch phaseType { + case AddsPhaseType: + hg.size.Add(hg.size, set.GetSize()) + hg.vertexAdds[shardKey] = set + case RemovesPhaseType: + hg.size.Sub(hg.size, set.GetSize()) + hg.vertexRemoves[shardKey] = set + } + case HyperedgeAtomType: + switch phaseType { + case AddsPhaseType: + hg.size.Add(hg.size, set.GetSize()) + hg.hyperedgeAdds[shardKey] = set + case RemovesPhaseType: + hg.size.Sub(hg.size, set.GetSize()) + hg.hyperedgeRemoves[shardKey] = set + } } return nil @@ -498,32 +507,18 @@ func (hg *Hypergraph) GetSize() *big.Int { } func (hg *Hypergraph) getOrCreateIdSet( - shardKey ShardKey, + shardAddr ShardKey, addMap map[ShardKey]*IdSet, removeMap map[ShardKey]*IdSet, atomType AtomType, ) (*IdSet, *IdSet) { - if _, ok := addMap[shardKey]; !ok { - phaseSet := protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS - if atomType == HyperedgeAtomType { - phaseSet = protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS - } - addMap[shardKey] = NewIdSet(shardKey, phaseSet, hg.treeConstructor( - shardKey, - phaseSet, - )) + if _, ok := addMap[shardAddr]; !ok { + addMap[shardAddr] = NewIdSet(atomType) } - if _, ok := removeMap[shardKey]; !ok { - phaseSet := protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES - if atomType == HyperedgeAtomType { - phaseSet = protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES - } - removeMap[shardKey] = NewIdSet(shardKey, phaseSet, hg.treeConstructor( - shardKey, - phaseSet, - )) + if _, ok := removeMap[shardAddr]; !ok { + removeMap[shardAddr] = NewIdSet(atomType) } - return addMap[shardKey], removeMap[shardKey] + return addMap[shardAddr], removeMap[shardAddr] } func (hg *Hypergraph) AddVertex(v Vertex) error { @@ -692,3 +687,37 @@ func (hg *Hypergraph) Within(a, h Atom) bool { } return false } + +func (hg *Hypergraph) GetReconciledVertexSetForShard( + shardKey ShardKey, +) *IdSet { + vertices := NewIdSet(VertexAtomType) + + if addSet, ok := hg.vertexAdds[shardKey]; ok { + removeSet := hg.vertexRemoves[shardKey] + for id, v := range addSet.atoms { + if !removeSet.Has(id) { + vertices.Add(v) + } + } + } + + return vertices +} + +func (hg *Hypergraph) GetReconciledHyperedgeSetForShard( + shardKey ShardKey, +) *IdSet { + hyperedges := NewIdSet(HyperedgeAtomType) + + if addSet, ok := hg.hyperedgeAdds[shardKey]; ok { + removeSet := hg.hyperedgeRemoves[shardKey] + for _, h := range addSet.atoms { + if !removeSet.Has(h.GetID()) { + hyperedges.Add(h) + } + } + } + + return hyperedges +} diff --git a/node/hypergraph/application/hypergraph_convergence_test.go b/node/hypergraph/application/hypergraph_convergence_test.go index 34cfefb..06340b5 100644 --- a/node/hypergraph/application/hypergraph_convergence_test.go +++ b/node/hypergraph/application/hypergraph_convergence_test.go @@ -10,11 +10,8 @@ import ( "time" "github.com/cloudflare/circl/sign/ed448" - "go.uber.org/zap" "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" - "source.quilibrium.com/quilibrium/monorepo/node/protobufs" - "source.quilibrium.com/quilibrium/monorepo/node/store" ) type Operation struct { @@ -25,13 +22,13 @@ type Operation struct { func TestConvergence(t *testing.T) { numParties := 4 - numOperations := 10000 + numOperations := 100000 enc := crypto.NewMPCitHVerifiableEncryptor(1) pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() vertices := make([]application.Vertex, numOperations) - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -80,19 +77,9 @@ func TestConvergence(t *testing.T) { } } - inmem := store.NewInMemKVDB() - logger, _ := zap.NewProduction() - hgStore := store.NewPebbleHypergraphStore(inmem, logger) - crdts := make([]*application.Hypergraph, numParties) for i := 0; i < numParties; i++ { - crdts[i] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - hgStore, - shardKey, - phaseSet, - ) - }) + crdts[i] = application.NewHypergraph() } for i := 0; i < numParties; i++ { diff --git a/node/hypergraph/application/hypergraph_test.go b/node/hypergraph/application/hypergraph_test.go index 1ff11cc..922e638 100644 --- a/node/hypergraph/application/hypergraph_test.go +++ b/node/hypergraph/application/hypergraph_test.go @@ -11,13 +11,10 @@ import ( "github.com/cloudflare/circl/sign/ed448" "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" - "source.quilibrium.com/quilibrium/monorepo/node/protobufs" ) func TestHypergraph(t *testing.T) { - hg := application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree { - return &crypto.RawVectorCommitmentTree{} - }) + hg := application.NewHypergraph() // Test vertex operations t.Run("Vertex Operations", func(t *testing.T) { @@ -25,7 +22,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -72,7 +69,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -116,7 +113,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -154,7 +151,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -189,7 +186,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -225,7 +222,7 @@ func TestHypergraph(t *testing.T) { pub, _, _ := ed448.GenerateKey(crand.Reader) data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) diff --git a/node/rpc/hypergraph_sync_rpc_server.go b/node/rpc/hypergraph_sync_rpc_server.go index 565dbe9..27914c8 100644 --- a/node/rpc/hypergraph_sync_rpc_server.go +++ b/node/rpc/hypergraph_sync_rpc_server.go @@ -43,7 +43,7 @@ func NewHypergraphComparisonServer( func sendLeafData( stream protobufs.HypergraphComparisonService_HyperStreamClient, hypergraphStore store.HypergraphStore, - localTree *crypto.RawVectorCommitmentTree, + localTree *crypto.VectorCommitmentTree, path []int32, metadataOnly bool, ) error { @@ -161,7 +161,7 @@ func getNodeAtPath( // getBranchInfoFromTree looks up the node at the given path in the local tree, // computes its commitment, and (if it is a branch) collects its immediate // children's commitments. -func getBranchInfoFromTree(tree *crypto.RawVectorCommitmentTree, path []int32) ( +func getBranchInfoFromTree(tree *crypto.VectorCommitmentTree, path []int32) ( *protobufs.HypergraphComparisonResponse, error, ) { @@ -207,7 +207,7 @@ func isLeaf(info *protobufs.HypergraphComparisonResponse) bool { func sendLeafDataServer( stream protobufs.HypergraphComparisonService_HyperStreamServer, hypergraphStore store.HypergraphStore, - localTree *crypto.RawVectorCommitmentTree, + localTree *crypto.VectorCommitmentTree, path []int32, metadataOnly bool, ) error { @@ -309,10 +309,7 @@ func syncTreeBidirectionallyServer( // Send our root branch info. rootPath := []int32{} - rootInfo, err := getBranchInfoFromTree( - idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(), - rootPath, - ) + rootInfo, err := getBranchInfoFromTree(idSet.GetTree(), rootPath) if err != nil { return err } @@ -377,7 +374,7 @@ func syncTreeBidirectionallyServer( zap.String("path", hex.EncodeToString(packNibbles(remoteInfo.Path))), ) localInfo, err := getBranchInfoFromTree( - idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(), + idSet.GetTree(), remoteInfo.Path, ) if err != nil { @@ -425,7 +422,7 @@ func syncTreeBidirectionallyServer( if err := sendLeafDataServer( stream, localHypergraphStore, - idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(), + idSet.GetTree(), remoteInfo.Path, metadataOnly, ); err != nil { @@ -480,7 +477,7 @@ func syncTreeBidirectionallyServer( if err := sendLeafDataServer( stream, localHypergraphStore, - idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(), + idSet.GetTree(), queryPath, metadataOnly, ); err != nil { @@ -494,10 +491,7 @@ func syncTreeBidirectionallyServer( hex.EncodeToString(packNibbles(queryPath)), ), ) - branchInfo, err := getBranchInfoFromTree( - idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(), - queryPath, - ) + branchInfo, err := getBranchInfoFromTree(idSet.GetTree(), queryPath) if err != nil { continue } @@ -524,7 +518,7 @@ func syncTreeBidirectionallyServer( if err != nil { return err } - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer b.Write(remoteUpdate.UnderlyingData) @@ -576,7 +570,7 @@ func SyncTreeBidirectionally( shardKey []byte, phaseSet protobufs.HypergraphPhaseSet, hypergraphStore store.HypergraphStore, - localTree crypto.VectorCommitmentTree, + localTree *crypto.VectorCommitmentTree, metadataOnly bool, ) error { logger.Info( @@ -599,10 +593,7 @@ func SyncTreeBidirectionally( } rootPath := []int32{} - rootInfo, err := getBranchInfoFromTree( - localTree.(*store.PersistentVectorTree).GetInternalTree(), - rootPath, - ) + rootInfo, err := getBranchInfoFromTree(localTree, rootPath) if err != nil { return err } @@ -662,10 +653,7 @@ func SyncTreeBidirectionally( "handling response", zap.String("path", hex.EncodeToString(packNibbles(remoteInfo.Path))), ) - localInfo, err := getBranchInfoFromTree( - localTree.(*store.PersistentVectorTree).GetInternalTree(), - remoteInfo.Path, - ) + localInfo, err := getBranchInfoFromTree(localTree, remoteInfo.Path) if err != nil { logger.Info( "requesting missing node", @@ -707,7 +695,7 @@ func SyncTreeBidirectionally( if err := sendLeafData( stream, hypergraphStore, - localTree.(*store.PersistentVectorTree).GetInternalTree(), + localTree, remoteInfo.Path, metadataOnly, ); err != nil { @@ -762,7 +750,7 @@ func SyncTreeBidirectionally( if err := sendLeafData( stream, hypergraphStore, - localTree.(*store.PersistentVectorTree).GetInternalTree(), + localTree, queryPath, metadataOnly, ); err != nil { @@ -776,10 +764,7 @@ func SyncTreeBidirectionally( hex.EncodeToString(packNibbles(queryPath)), ), ) - branchInfo, err := getBranchInfoFromTree( - localTree.(*store.PersistentVectorTree).GetInternalTree(), - queryPath, - ) + branchInfo, err := getBranchInfoFromTree(localTree, queryPath) if err != nil { continue } @@ -807,7 +792,7 @@ func SyncTreeBidirectionally( if err != nil { return err } - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer b.Write(remoteUpdate.UnderlyingData) diff --git a/node/rpc/hypergraph_sync_rpc_server_test.go b/node/rpc/hypergraph_sync_rpc_server_test.go index cf7a2af..2ec8ab9 100644 --- a/node/rpc/hypergraph_sync_rpc_server_test.go +++ b/node/rpc/hypergraph_sync_rpc_server_test.go @@ -12,7 +12,6 @@ import ( "testing" "github.com/cloudflare/circl/sign/ed448" - "github.com/stretchr/testify/assert" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -38,7 +37,7 @@ func TestHypergraphSyncServer(t *testing.T) { data := enc.Encrypt(make([]byte, 20), pub) verenc := data[0].Compress() vertices := make([]application.Vertex, numOperations) - dataTree := &crypto.RawVectorCommitmentTree{} + dataTree := &crypto.VectorCommitmentTree{} for _, d := range []application.Encrypted{verenc} { dataBytes := d.ToBytes() id := sha512.Sum512(dataBytes) @@ -89,35 +88,15 @@ func TestHypergraphSyncServer(t *testing.T) { } } - serverKvdb := store.NewInMemKVDB() clientKvdb := store.NewInMemKVDB() - controlKvdb := store.NewInMemKVDB() + serverKvdb := store.NewInMemKVDB() logger, _ := zap.NewProduction() - serverHypergraphStore := store.NewPebbleHypergraphStore(serverKvdb, logger) clientHypergraphStore := store.NewPebbleHypergraphStore(clientKvdb, logger) - controlHypergraphStore := store.NewPebbleHypergraphStore(controlKvdb, logger) + serverHypergraphStore := store.NewPebbleHypergraphStore(serverKvdb, logger) crdts := make([]*application.Hypergraph, numParties) - crdts[0] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - serverHypergraphStore, - shardKey, - phaseSet, - ) - }) - crdts[1] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - clientHypergraphStore, - shardKey, - phaseSet, - ) - }) - crdts[2] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree { - return store.NewPersistentVectorTree( - controlHypergraphStore, - shardKey, - phaseSet, - ) - }) + for i := 0; i < numParties; i++ { + crdts[i] = application.NewHypergraph() + } txn, _ := serverHypergraphStore.NewTransaction(false) for _, op := range operations1[:5000] { @@ -135,7 +114,7 @@ func TestHypergraphSyncServer(t *testing.T) { } } txn.Commit() - for _, op := range operations2[:5000] { + for _, op := range operations2[:500] { switch op.Type { case "AddVertex": crdts[0].AddVertex(op.Vertex) @@ -164,7 +143,7 @@ func TestHypergraphSyncServer(t *testing.T) { } } txn.Commit() - for _, op := range operations2[5000:] { + for _, op := range operations2[500:] { switch op.Type { case "AddVertex": crdts[1].AddVertex(op.Vertex) @@ -205,31 +184,7 @@ func TestHypergraphSyncServer(t *testing.T) { crdts[0].Commit() crdts[1].Commit() crdts[2].Commit() - txn, _ = serverHypergraphStore.NewTransaction(false) - serverHypergraphStore.SaveHypergraph(txn, crdts[0]) - txn.Commit() - txn, _ = clientHypergraphStore.NewTransaction(false) - clientHypergraphStore.SaveHypergraph(txn, crdts[1]) - txn.Commit() - txn, _ = controlHypergraphStore.NewTransaction(false) - controlHypergraphStore.SaveHypergraph(txn, crdts[2]) - txn.Commit() - var err error - eval0, err := serverHypergraphStore.LoadHypergraph() - assert.NoError(t, err) - eval1, err := clientHypergraphStore.LoadHypergraph() - assert.NoError(t, err) log.Printf("Generated data") - leaves0 := crypto.CompareLeaves( - crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - eval0.GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - ) - leaves1 := crypto.CompareLeaves( - crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - eval1.GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - ) - assert.Len(t, leaves0, 0) - assert.Len(t, leaves1, 0) lis, err := net.Listen("tcp", ":50051") if err != nil { @@ -257,22 +212,14 @@ func TestHypergraphSyncServer(t *testing.T) { log.Fatalf("Client: failed to stream: %v", err) } - err = rpc.SyncTreeBidirectionally( - str, - logger, - append(append([]byte{}, shardKey.L1[:]...), shardKey.L2[:]...), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS, - clientHypergraphStore, - crdts[1].GetVertexAdds()[shardKey].GetTree(), - false, - ) + err = rpc.SyncTreeBidirectionally(str, logger, append(append([]byte{}, shardKey.L1[:]...), shardKey.L2[:]...), protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS, clientHypergraphStore, crdts[1].GetVertexAdds()[shardKey].GetTree(), false) if err != nil { log.Fatalf("Client: failed to sync 1: %v", err) } leaves := crypto.CompareLeaves( - crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), + crdts[0].GetVertexAdds()[shardKey].GetTree(), + crdts[1].GetVertexAdds()[shardKey].GetTree(), ) fmt.Println(len(leaves)) @@ -294,8 +241,8 @@ func TestHypergraphSyncServer(t *testing.T) { crdts[1].GetVertexAdds()[shardKey].GetTree().Commit(false), ) { leaves := crypto.CompareLeaves( - crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), - crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(), + crdts[0].GetVertexAdds()[shardKey].GetTree(), + crdts[1].GetVertexAdds()[shardKey].GetTree(), ) fmt.Println(len(leaves)) log.Fatalf( diff --git a/node/store/clock.go b/node/store/clock.go index 089a109..0ffa079 100644 --- a/node/store/clock.go +++ b/node/store/clock.go @@ -104,11 +104,11 @@ type ClockStore interface { minFrameNumber uint64, maxFrameNumber uint64, ) error - GetDataStateTree(filter []byte) (*crypto.RawVectorCommitmentTree, error) + GetDataStateTree(filter []byte) (*crypto.VectorCommitmentTree, error) SetDataStateTree( txn Transaction, filter []byte, - tree *crypto.RawVectorCommitmentTree, + tree *crypto.VectorCommitmentTree, ) error } @@ -1696,7 +1696,7 @@ func (p *PebbleClockStore) SetProverTriesForFrame( } func (p *PebbleClockStore) GetDataStateTree(filter []byte) ( - *crypto.RawVectorCommitmentTree, + *crypto.VectorCommitmentTree, error, ) { data, closer, err := p.db.Get(clockDataStateTreeKey(filter)) @@ -1708,7 +1708,7 @@ func (p *PebbleClockStore) GetDataStateTree(filter []byte) ( return nil, errors.Wrap(err, "get data state tree") } defer closer.Close() - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer b.Write(data) dec := gob.NewDecoder(&b) @@ -1722,7 +1722,7 @@ func (p *PebbleClockStore) GetDataStateTree(filter []byte) ( func (p *PebbleClockStore) SetDataStateTree( txn Transaction, filter []byte, - tree *crypto.RawVectorCommitmentTree, + tree *crypto.VectorCommitmentTree, ) error { b := new(bytes.Buffer) enc := gob.NewEncoder(b) diff --git a/node/store/hypergraph.go b/node/store/hypergraph.go index 3156859..b795669 100644 --- a/node/store/hypergraph.go +++ b/node/store/hypergraph.go @@ -3,33 +3,30 @@ package store import ( "bytes" "encoding/gob" - "fmt" - "math/big" "github.com/pkg/errors" "go.uber.org/zap" "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" - "source.quilibrium.com/quilibrium/monorepo/node/protobufs" ) type HypergraphStore interface { NewTransaction(indexed bool) (Transaction, error) LoadVertexTree(id []byte) ( - *crypto.RawVectorCommitmentTree, + *crypto.VectorCommitmentTree, error, ) LoadVertexData(id []byte) ([]application.Encrypted, error) SaveVertexTree( txn Transaction, id []byte, - vertTree *crypto.RawVectorCommitmentTree, + vertTree *crypto.VectorCommitmentTree, ) error CommitAndSaveVertexData( txn Transaction, id []byte, data []application.Encrypted, - ) (*crypto.RawVectorCommitmentTree, []byte, error) + ) (*crypto.VectorCommitmentTree, []byte, error) LoadHypergraph() ( *application.Hypergraph, error, @@ -38,14 +35,6 @@ type HypergraphStore interface { txn Transaction, hg *application.Hypergraph, ) error - GetBranchNode(id NodeID) (*StoredBranchNode, error) - GetLeafNode(id NodeID) (*StoredLeafNode, error) - BatchWrite( - txn Transaction, - branches map[NodeID]*StoredBranchNode, - leaves map[NodeID]*StoredLeafNode, - deletions map[NodeID]struct{}, - ) error } var _ HypergraphStore = (*PebbleHypergraphStore)(nil) @@ -72,9 +61,6 @@ const ( VERTEX_DATA = 0xF0 HYPEREDGE_ADDS = 0x01 HYPEREDGE_REMOVES = 0x11 - SET_TREE_ROOT = 0x00 - SET_TREE_BRANCH = 0x01 - SET_TREE_LEAF = 0x02 ) func hypergraphVertexAddsKey(shardKey application.ShardKey) []byte { @@ -113,44 +99,11 @@ func hypergraphHyperedgeRemovesKey(shardKey application.ShardKey) []byte { func shardKeyFromKey(key []byte) application.ShardKey { return application.ShardKey{ - L1: [3]byte(key[3:6]), - L2: [32]byte(key[6:]), + L1: [3]byte(key[2:5]), + L2: [32]byte(key[5:]), } } -func SetTreeBranchKey( - shardKey application.ShardKey, - phaseSet byte, - prefix []int, -) NodeID { - key := []byte{HYPERGRAPH_SHARD, phaseSet} - if len(prefix) == 0 { - key = append(key, SET_TREE_ROOT) - key = append(key, shardKey.L1[:]...) - key = append(key, shardKey.L2[:]...) - return NodeID(key) - } - - key = append(key, SET_TREE_BRANCH) - key = append(key, shardKey.L1[:]...) - key = append(key, shardKey.L2[:]...) - key = append(key, packNibbles(prefix)...) - return NodeID(key) -} - -func SetTreeLeafKey( - shardKey application.ShardKey, - phaseSet byte, - leafKey []byte, -) NodeID { - key := []byte{HYPERGRAPH_SHARD, phaseSet} - key = append(key, SET_TREE_LEAF) - key = append(key, shardKey.L1[:]...) - key = append(key, shardKey.L2[:]...) - key = append(key, leafKey...) - return NodeID(key) -} - func (p *PebbleHypergraphStore) NewTransaction(indexed bool) ( Transaction, error, @@ -159,10 +112,10 @@ func (p *PebbleHypergraphStore) NewTransaction(indexed bool) ( } func (p *PebbleHypergraphStore) LoadVertexTree(id []byte) ( - *crypto.RawVectorCommitmentTree, + *crypto.VectorCommitmentTree, error, ) { - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer vertexData, closer, err := p.db.Get(hypergraphVertexDataKey(id)) if err != nil { @@ -183,7 +136,7 @@ func (p *PebbleHypergraphStore) LoadVertexData(id []byte) ( []application.Encrypted, error, ) { - tree := &crypto.RawVectorCommitmentTree{} + tree := &crypto.VectorCommitmentTree{} var b bytes.Buffer vertexData, closer, err := p.db.Get(hypergraphVertexDataKey(id)) if err != nil { @@ -209,7 +162,7 @@ func (p *PebbleHypergraphStore) LoadVertexData(id []byte) ( func (p *PebbleHypergraphStore) SaveVertexTree( txn Transaction, id []byte, - vertTree *crypto.RawVectorCommitmentTree, + vertTree *crypto.VectorCommitmentTree, ) error { var buf bytes.Buffer enc := gob.NewEncoder(&buf) @@ -227,7 +180,7 @@ func (p *PebbleHypergraphStore) CommitAndSaveVertexData( txn Transaction, id []byte, data []application.Encrypted, -) (*crypto.RawVectorCommitmentTree, []byte, error) { +) (*crypto.VectorCommitmentTree, []byte, error) { dataTree := application.EncryptedToVertexTree(data) commit := dataTree.Commit(false) @@ -247,17 +200,10 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( *application.Hypergraph, error, ) { - hg := application.NewHypergraph( - func( - shardKey application.ShardKey, - phaseSet protobufs.HypergraphPhaseSet, - ) crypto.VectorCommitmentTree { - return NewPersistentVectorTree(p, shardKey, phaseSet) - }, - ) + hg := application.NewHypergraph() vertexAddsIter, err := p.db.NewIter( - []byte{HYPERGRAPH_SHARD, VERTEX_ADDS, SET_TREE_ROOT}, - []byte{HYPERGRAPH_SHARD, VERTEX_ADDS, SET_TREE_BRANCH}, + []byte{HYPERGRAPH_SHARD, VERTEX_ADDS}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES}, ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -266,30 +212,12 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( for vertexAddsIter.First(); vertexAddsIter.Valid(); vertexAddsIter.Next() { shardKey := make([]byte, len(vertexAddsIter.Key())) copy(shardKey, vertexAddsIter.Key()) - node := &StoredBranchNode{} - var b bytes.Buffer - b.Write(vertexAddsIter.Value()) - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - tree := NewPersistentVectorTree( - p, + err := hg.ImportFromBytes( + application.VertexAtomType, + application.AddsPhaseType, shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS, - ) - - tree.tree.Root, err = tree.storedToBranch(node) - if err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - err := hg.SetIdSet( - shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS, - tree, + vertexAddsIter.Value(), ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -297,8 +225,8 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( } vertexRemovesIter, err := p.db.NewIter( - []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES, SET_TREE_ROOT}, - []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES, SET_TREE_BRANCH}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES + 1}, ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -307,30 +235,12 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( for vertexRemovesIter.First(); vertexRemovesIter.Valid(); vertexRemovesIter.Next() { shardKey := make([]byte, len(vertexRemovesIter.Key())) copy(shardKey, vertexRemovesIter.Key()) - node := &StoredBranchNode{} - var b bytes.Buffer - b.Write(vertexRemovesIter.Value()) - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - tree := NewPersistentVectorTree( - p, + err := hg.ImportFromBytes( + application.VertexAtomType, + application.RemovesPhaseType, shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES, - ) - - tree.tree.Root, err = tree.storedToBranch(node) - if err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - err := hg.SetIdSet( - shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES, - tree, + vertexRemovesIter.Value(), ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -338,8 +248,8 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( } hyperedgeAddsIter, err := p.db.NewIter( - []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS, SET_TREE_ROOT}, - []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS, SET_TREE_BRANCH}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES}, ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -348,30 +258,12 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( for hyperedgeAddsIter.First(); hyperedgeAddsIter.Valid(); hyperedgeAddsIter.Next() { shardKey := make([]byte, len(hyperedgeAddsIter.Key())) copy(shardKey, hyperedgeAddsIter.Key()) - node := &StoredBranchNode{} - var b bytes.Buffer - b.Write(hyperedgeAddsIter.Value()) - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - tree := NewPersistentVectorTree( - p, + err := hg.ImportFromBytes( + application.HyperedgeAtomType, + application.AddsPhaseType, shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS, - ) - - tree.tree.Root, err = tree.storedToBranch(node) - if err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - err := hg.SetIdSet( - shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS, - tree, + hyperedgeAddsIter.Value(), ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -379,8 +271,8 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( } hyperedgeRemovesIter, err := p.db.NewIter( - []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES, SET_TREE_ROOT}, - []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES, SET_TREE_BRANCH}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES + 1}, ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -389,30 +281,12 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( for hyperedgeRemovesIter.First(); hyperedgeRemovesIter.Valid(); hyperedgeRemovesIter.Next() { shardKey := make([]byte, len(hyperedgeRemovesIter.Key())) copy(shardKey, hyperedgeRemovesIter.Key()) - node := &StoredBranchNode{} - var b bytes.Buffer - b.Write(hyperedgeRemovesIter.Value()) - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - tree := NewPersistentVectorTree( - p, + err := hg.ImportFromBytes( + application.HyperedgeAtomType, + application.RemovesPhaseType, shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES, - ) - - tree.tree.Root, err = tree.storedToBranch(node) - if err != nil { - return nil, errors.Wrap(err, "load hypergraph") - } - - err := hg.SetIdSet( - shardKeyFromKey(shardKey), - protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES, - tree, + hyperedgeRemovesIter.Value(), ) if err != nil { return nil, errors.Wrap(err, "load hypergraph") @@ -426,38 +300,45 @@ func (p *PebbleHypergraphStore) SaveHypergraph( txn Transaction, hg *application.Hypergraph, ) error { - hg.Commit() - - for _, vertexAdds := range hg.GetVertexAdds() { + for shardKey, vertexAdds := range hg.GetVertexAdds() { if vertexAdds.IsDirty() { - err := vertexAdds.GetTree().(*PersistentVectorTree).WriteBatch(txn) + err := txn.Set(hypergraphVertexAddsKey(shardKey), vertexAdds.ToBytes()) if err != nil { return errors.Wrap(err, "save hypergraph") } } } - for _, vertexRemoves := range hg.GetVertexRemoves() { + for shardKey, vertexRemoves := range hg.GetVertexRemoves() { if vertexRemoves.IsDirty() { - err := vertexRemoves.GetTree().(*PersistentVectorTree).WriteBatch(txn) + err := txn.Set( + hypergraphVertexRemovesKey(shardKey), + vertexRemoves.ToBytes(), + ) if err != nil { return errors.Wrap(err, "save hypergraph") } } } - for _, hyperedgeAdds := range hg.GetHyperedgeAdds() { + for shardKey, hyperedgeAdds := range hg.GetHyperedgeAdds() { if hyperedgeAdds.IsDirty() { - err := hyperedgeAdds.GetTree().(*PersistentVectorTree).WriteBatch(txn) + err := txn.Set( + hypergraphHyperedgeAddsKey(shardKey), + hyperedgeAdds.ToBytes(), + ) if err != nil { return errors.Wrap(err, "save hypergraph") } } } - for _, hyperedgeRemoves := range hg.GetHyperedgeRemoves() { + for shardKey, hyperedgeRemoves := range hg.GetHyperedgeRemoves() { if hyperedgeRemoves.IsDirty() { - err := hyperedgeRemoves.GetTree().(*PersistentVectorTree).WriteBatch(txn) + err := txn.Set( + hypergraphHyperedgeRemovesKey(shardKey), + hyperedgeRemoves.ToBytes(), + ) if err != nil { return errors.Wrap(err, "save hypergraph") } @@ -466,798 +347,3 @@ func (p *PebbleHypergraphStore) SaveHypergraph( return nil } - -func (p *PebbleHypergraphStore) GetBranchNode(id NodeID) ( - *StoredBranchNode, - error, -) { - data, closer, err := p.db.Get([]byte(id)) - if err != nil { - return nil, errors.Wrap(err, "get branch node") - } - defer closer.Close() - - node := &StoredBranchNode{} - var b bytes.Buffer - b.Write(data) - - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "get branch node") - } - - return node, nil -} - -func (p *PebbleHypergraphStore) GetLeafNode(id NodeID) ( - *StoredLeafNode, - error, -) { - data, closer, err := p.db.Get([]byte(id)) - if err != nil { - return nil, errors.Wrap(err, "get branch node") - } - defer closer.Close() - - node := &StoredLeafNode{} - var b bytes.Buffer - b.Write(data) - - dec := gob.NewDecoder(&b) - if err := dec.Decode(node); err != nil { - return nil, errors.Wrap(err, "get branch node") - } - - return node, nil -} - -func (p *PebbleHypergraphStore) BatchWrite( - txn Transaction, - branches map[NodeID]*StoredBranchNode, - leaves map[NodeID]*StoredLeafNode, - deletions map[NodeID]struct{}, -) error { - for id := range deletions { - if err := txn.Delete([]byte(id)); err != nil { - return errors.Wrap(err, "batch write") - } - } - - for id, node := range branches { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - if err := enc.Encode(node); err != nil { - return errors.Wrap(err, "batch write") - } - - if err := txn.Set([]byte(id), buf.Bytes()); err != nil { - return errors.Wrap(err, "batch write") - } - } - - for id, node := range leaves { - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - if err := enc.Encode(node); err != nil { - return errors.Wrap(err, "batch write") - } - - if err := txn.Set([]byte(id), buf.Bytes()); err != nil { - return errors.Wrap(err, "batch write") - } - } - - return nil -} - -// StoredBranchNode represents the serializable form of a branch node. -// Storage of trees necessarily includes the metadata because trees can be -// lazily loaded, and this metadata is sometimes the only information we need. -// Commitments should also be stored due to time cost of calculation. -type StoredBranchNode struct { - Prefix []int - Commitment []byte - ChildrenIDs [crypto.NodesPerBranch]NodeID - Size []byte - LeafCount int - LongestBranch int -} - -// StoredLeafNode represents the serializable form of a leaf node. -// Storage of leaves necessarily includes the value of the data as we may not -// be dynamically resolving the underlying indirect reference the value -// represents with lazy loading, but the distinction between the value and hash -// target (if present) may be important. -type StoredLeafNode struct { - Commitment []byte - Key []byte - Value []byte - HashTarget []byte - Size []byte -} - -type NodeID string - -type PersistentVectorTree struct { - store HypergraphStore - tree *crypto.RawVectorCommitmentTree - shardKey application.ShardKey - phaseSet byte - addedBranches map[NodeID]*StoredBranchNode - addedLeaves map[NodeID]*StoredLeafNode - deletions map[NodeID]struct{} -} - -var _ crypto.VectorCommitmentTree = (*PersistentVectorTree)(nil) - -func NewPersistentVectorTree( - store HypergraphStore, - shardKey application.ShardKey, - phaseSet protobufs.HypergraphPhaseSet, -) *PersistentVectorTree { - phaseByte := byte(0x00) - switch phaseSet { - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS: - phaseByte = VERTEX_ADDS - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES: - phaseByte = VERTEX_REMOVES - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS: - phaseByte = HYPEREDGE_ADDS - case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES: - phaseByte = HYPEREDGE_REMOVES - } - - return &PersistentVectorTree{ - store: store, - tree: &crypto.RawVectorCommitmentTree{}, - shardKey: shardKey, - phaseSet: phaseByte, - addedBranches: make(map[NodeID]*StoredBranchNode), - addedLeaves: make(map[NodeID]*StoredLeafNode), - deletions: make(map[NodeID]struct{}), - } -} - -func serializeBigInt(n *big.Int) []byte { - if n == nil { - return nil - } - return n.FillBytes(make([]byte, 32)) -} - -func deserializeBigInt(data []byte) *big.Int { - if len(data) == 0 { - return big.NewInt(0) - } - return new(big.Int).SetBytes(data) -} - -func branchToStored( - shardKey application.ShardKey, - phaseSet byte, - prefix []int, - node *crypto.VectorCommitmentBranchNode, -) *StoredBranchNode { - stored := &StoredBranchNode{ - Prefix: make([]int, len(node.Prefix)), - Commitment: node.Commitment, - ChildrenIDs: [crypto.NodesPerBranch]NodeID{}, - Size: serializeBigInt(node.Size), - LeafCount: node.LeafCount, - LongestBranch: node.LongestBranch, - } - copy(stored.Prefix, node.Prefix) - for i, child := range node.Children { - if child == nil { - continue - } - switch c := child.(type) { - case *crypto.VectorCommitmentBranchNode: - stored.ChildrenIDs[i] = SetTreeBranchKey( - shardKey, - phaseSet, - append(append(append([]int{}, prefix...), node.Prefix...), i), - ) - case *crypto.VectorCommitmentLeafNode: - stored.ChildrenIDs[i] = SetTreeLeafKey( - shardKey, - phaseSet, - c.Key, - ) - } - } - return stored -} - -func leafToStored(node *crypto.VectorCommitmentLeafNode) *StoredLeafNode { - return &StoredLeafNode{ - Key: node.Key, - Commitment: node.Commitment, - Value: node.Value, - HashTarget: node.HashTarget, - Size: serializeBigInt(node.Size), - } -} - -func (t *PersistentVectorTree) storedToBranch( - stored *StoredBranchNode, -) (*crypto.VectorCommitmentBranchNode, error) { - node := &crypto.VectorCommitmentBranchNode{ - Prefix: stored.Prefix, - Commitment: stored.Commitment, - Children: [crypto.NodesPerBranch]crypto.VectorCommitmentNode{}, - Size: deserializeBigInt(stored.Size), - LeafCount: stored.LeafCount, - LongestBranch: stored.LongestBranch, - } - - for i, childID := range stored.ChildrenIDs { - if childID == "" { - continue - } - var child crypto.VectorCommitmentNode - var err error - - if childID[2] == SET_TREE_BRANCH { - child, err = t.loadBranchNode(childID) - } else { - child, err = t.loadLeafNode(childID) - } - if err != nil { - return nil, err - } - node.Children[i] = child - } - - return node, nil -} - -func storedToLeaf(stored *StoredLeafNode) *crypto.VectorCommitmentLeafNode { - return &crypto.VectorCommitmentLeafNode{ - Key: stored.Key, - Commitment: stored.Commitment, - Value: stored.Value, - HashTarget: stored.HashTarget, - Size: deserializeBigInt(stored.Size), - } -} - -func (t *PersistentVectorTree) loadBranchNode( - id NodeID, -) (*crypto.VectorCommitmentBranchNode, error) { - stored, err := t.store.GetBranchNode(id) - if err != nil { - return nil, fmt.Errorf("failed to load branch node %s: %w", id, err) - } - return t.storedToBranch(stored) -} - -func (t *PersistentVectorTree) loadLeafNode(id NodeID) ( - *crypto.VectorCommitmentLeafNode, - error, -) { - stored, err := t.store.GetLeafNode(id) - if err != nil { - return nil, errors.Wrap( - errors.Wrap( - err, - fmt.Sprintf("failed to load leaf node %s", id), - ), - "load leaf node", - ) - } - return storedToLeaf(stored), nil -} - -func (t *PersistentVectorTree) Load() error { - stored, err := t.store.GetBranchNode(SetTreeBranchKey( - t.shardKey, - t.phaseSet, - []int{}, - )) - if err != nil { - return err - } - - root, err := t.storedToBranch(stored) - if err != nil { - return err - } - - t.tree.Root = root - return nil -} - -func (t *PersistentVectorTree) trackNodeChanges( - oldPrefix, newPrefix []int, - oldNode, newNode crypto.VectorCommitmentNode, -) { - if oldNode == nil && newNode == nil { - return - } - - // deletions first - if oldNode != nil { - var oldID NodeID - switch n := oldNode.(type) { - case *crypto.VectorCommitmentBranchNode: - oldID = SetTreeBranchKey( - t.shardKey, - t.phaseSet, - oldPrefix, - ) - case *crypto.VectorCommitmentLeafNode: - oldID = SetTreeLeafKey( - t.shardKey, - t.phaseSet, - n.Key, - ) - } - - if _, ok := t.addedBranches[oldID]; ok { - delete(t.addedBranches, oldID) - } else if _, ok := t.addedLeaves[oldID]; ok { - delete(t.addedLeaves, oldID) - } else { - t.deletions[oldID] = struct{}{} - } - } - - // then additions - if newNode != nil { - switch n := newNode.(type) { - case *crypto.VectorCommitmentBranchNode: - id := SetTreeBranchKey( - t.shardKey, - t.phaseSet, - newPrefix, - ) - t.addedBranches[id] = branchToStored( - t.shardKey, - t.phaseSet, - newPrefix, - n, - ) - case *crypto.VectorCommitmentLeafNode: - id := SetTreeLeafKey( - t.shardKey, - t.phaseSet, - n.Key, - ) - t.addedLeaves[id] = leafToStored(n) - } - } -} - -func (t *PersistentVectorTree) Get(key []byte) ([]byte, error) { - if len(key) == 0 { - return nil, errors.New("empty key not allowed") - } - - var get func( - prefix []int, - node crypto.VectorCommitmentNode, - depth int, - ) ([]byte, error) - - get = func( - prefix []int, - node crypto.VectorCommitmentNode, - depth int, - ) ([]byte, error) { - if node == nil { - return nil, nil - } - - switch n := node.(type) { - case *crypto.VectorCommitmentLeafNode: - if bytes.Equal(n.Key, key) { - return n.Value, nil - } - return nil, nil - - case *crypto.VectorCommitmentBranchNode: - // Check prefix match - for i, expectedNibble := range n.Prefix { - if crypto.GetNextNibble( - key, - depth+i*crypto.BranchBits, - ) != expectedNibble { - return nil, nil - } - } - // Get final nibble after prefix - finalNibble := crypto.GetNextNibble( - key, - depth+len(n.Prefix)*crypto.BranchBits, - ) - - isLoaded := false - for _, c := range n.Children { - if c != nil { - isLoaded = true - break - } - } - - if !isLoaded { - var self *crypto.VectorCommitmentBranchNode - var err error - - if len(prefix) == 0 { - self, err = t.loadBranchNode( - SetTreeBranchKey( - t.shardKey, - t.phaseSet, - append([]int{}, prefix...), - ), - ) - } else { - self, err = t.loadBranchNode( - SetTreeBranchKey( - t.shardKey, - t.phaseSet, - append( - append([]int{}, prefix...), - n.Prefix..., - ), - ), - ) - } - - if err != nil { - return nil, errors.Wrap(err, "get") - } - n.Children = self.Children - } - return get( - append( - append( - append([]int{}, prefix...), - n.Prefix..., - ), - finalNibble, - ), - n.Children[finalNibble], - depth+len(n.Prefix)*crypto.BranchBits+crypto.BranchBits, - ) - } - - return nil, nil - } - - value, err := get([]int{}, t.tree.Root, 0) - if err != nil { - return nil, err - } - - if value == nil { - return nil, errors.New("key not found") - } - - return value, nil -} - -func (t *PersistentVectorTree) Insert( - key, value, hashTarget []byte, - size *big.Int, -) error { - if len(key) == 0 { - return errors.New("empty key not allowed") - } - - var insert func( - prefix []int, - node crypto.VectorCommitmentNode, - depth int, - ) (crypto.VectorCommitmentNode, error) - - insert = func( - prefix []int, - node crypto.VectorCommitmentNode, - depth int, - ) (crypto.VectorCommitmentNode, error) { - if node == nil { - t.trackNodeChanges( - nil, - prefix, - nil, - &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - }, - ) - return &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - }, nil - } - - switch n := node.(type) { - case *crypto.VectorCommitmentLeafNode: - if bytes.Equal(n.Key, key) { - n.Value = value - n.HashTarget = hashTarget - n.Commitment = nil - n.Size = size - t.trackNodeChanges(nil, prefix, nil, n) - return n, nil - } - - // Get common prefix nibbles and divergence point - sharedNibbles, divergeDepth := crypto.GetNibblesUntilDiverge( - n.Key, - key, - depth, - ) - - // Create single branch node with shared prefix - branch := &crypto.VectorCommitmentBranchNode{ - Prefix: sharedNibbles, - LeafCount: 2, - LongestBranch: 1, - Size: new(big.Int).Add(n.Size, size), - } - - // Add both leaves at their final positions - finalOldNibble := crypto.GetNextNibble(n.Key, divergeDepth) - finalNewNibble := crypto.GetNextNibble(key, divergeDepth) - branch.Children[finalOldNibble] = n - branch.Children[finalNewNibble] = &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - } - t.trackNodeChanges( - nil, - append(append(append([]int{}, prefix...), sharedNibbles...), finalOldNibble), - nil, - n, - ) - t.trackNodeChanges( - nil, - append(append(append([]int{}, prefix...), sharedNibbles...), finalNewNibble), - nil, - &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - }, - ) - t.trackNodeChanges( - nil, - prefix, - nil, - branch, - ) - return branch, nil - - case *crypto.VectorCommitmentBranchNode: - isLoaded := false - for _, c := range n.Children { - if c != nil { - isLoaded = true - break - } - } - - if !isLoaded { - self, err := t.loadBranchNode( - SetTreeBranchKey( - t.shardKey, - t.phaseSet, - append( - append([]int{}, prefix...), - n.Prefix..., - ), - ), - ) - if err != nil { - return nil, errors.Wrap(err, "insert") - } - n.Children = self.Children - } - - if len(n.Prefix) > 0 { - // Check if the new key matches the prefix - for i, expectedNibble := range n.Prefix { - actualNibble := crypto.GetNextNibble(key, depth+i*crypto.BranchBits) - if actualNibble != expectedNibble { - // Create new branch with shared prefix subset - newBranch := &crypto.VectorCommitmentBranchNode{ - Prefix: n.Prefix[:i], - LeafCount: n.LeafCount + 1, - LongestBranch: n.LongestBranch + 1, - Size: new(big.Int).Add(n.Size, size), - } - // Position old branch and new leaf - newBranch.Children[expectedNibble] = n - n.Prefix = n.Prefix[i+1:] // remove shared prefix from old branch - newBranch.Children[actualNibble] = &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - } - - t.trackNodeChanges( - prefix, - append(append(append([]int{}, prefix...), newBranch.Prefix...), expectedNibble), - n, - n, - ) - t.trackNodeChanges( - nil, - append(append(append([]int{}, prefix...), newBranch.Prefix...), actualNibble), - nil, - &crypto.VectorCommitmentLeafNode{ - Key: key, - Value: value, - HashTarget: hashTarget, - Size: size, - }, - ) - t.trackNodeChanges( - nil, - prefix, - nil, - newBranch, - ) - return newBranch, nil - } - } - - // Key matches prefix, continue with final nibble - finalNibble := crypto.GetNextNibble( - key, - depth+len(n.Prefix)*crypto.BranchBits, - ) - inserted, err := insert( - append( - append( - append([]int{}, prefix...), - n.Prefix..., - ), - finalNibble, - ), - n.Children[finalNibble], - depth+len(n.Prefix)*crypto.BranchBits+crypto.BranchBits, - ) - if err != nil { - return nil, err - } - n.Children[finalNibble] = inserted - n.Commitment = nil - n.LeafCount += 1 - switch i := inserted.(type) { - case *crypto.VectorCommitmentBranchNode: - if n.LongestBranch <= i.LongestBranch { - n.LongestBranch = i.LongestBranch + 1 - } - case *crypto.VectorCommitmentLeafNode: - n.LongestBranch = 1 - } - n.Size = n.Size.Add(n.Size, size) - - t.trackNodeChanges( - nil, - prefix, - nil, - n, - ) - return n, nil - } else { - // Simple branch without prefix - nibble := crypto.GetNextNibble(key, depth) - inserted, err := insert( - append( - append( - append([]int{}, prefix...), - n.Prefix..., - ), - nibble, - ), - n.Children[nibble], - depth+crypto.BranchBits, - ) - if err != nil { - return nil, err - } - n.Children[nibble] = inserted - n.Commitment = nil - n.LeafCount += 1 - switch i := inserted.(type) { - case *crypto.VectorCommitmentBranchNode: - if n.LongestBranch <= i.LongestBranch { - n.LongestBranch = i.LongestBranch + 1 - } - case *crypto.VectorCommitmentLeafNode: - n.LongestBranch = 1 - } - n.Size = n.Size.Add(n.Size, size) - - t.trackNodeChanges( - nil, - prefix, - nil, - n, - ) - return n, nil - } - } - - return nil, nil - } - - newRoot, err := insert([]int{}, t.tree.Root, 0) - if err != nil { - return err - } - - t.tree.Root = newRoot - - return nil -} - -func (t *PersistentVectorTree) Delete(key []byte) error { - return errors.New("deletion not supported") -} - -func (t *PersistentVectorTree) Commit(recalculate bool) []byte { - return t.tree.Commit(recalculate) -} - -func (t *PersistentVectorTree) WriteBatch(txn Transaction) error { - err := t.store.BatchWrite(txn, t.addedBranches, t.addedLeaves, t.deletions) - if err != nil { - return errors.Wrap(err, "write batch") - } - - // Reset change tracking - t.addedBranches = make(map[NodeID]*StoredBranchNode) - t.addedLeaves = make(map[NodeID]*StoredLeafNode) - t.deletions = make(map[NodeID]struct{}) - - return nil -} - -func (t *PersistentVectorTree) GetMetadata() ( - leafCount int, - longestBranch int, -) { - return t.tree.GetMetadata() -} - -func (t *PersistentVectorTree) GetSize() *big.Int { - return t.tree.GetSize() -} - -func (t *PersistentVectorTree) Prove(key []byte) [][]byte { - return t.tree.Prove(key) -} - -func (t *PersistentVectorTree) Verify(key []byte, proofs [][]byte) bool { - return t.tree.Verify(key, proofs) -} - -func ( - t *PersistentVectorTree, -) GetInternalTree() *crypto.RawVectorCommitmentTree { - return t.tree -} - -// this method is truncating the ints to bytes because the total size of the -// branch bits is 6. if this ever increases, this will break a lot of things. -func packNibbles(values []int) []byte { - out := []byte{} - for _, v := range values { - out = append(out, byte(v)) - } - return out -}