mirror of
https://github.com/QuilibriumNetwork/ceremonyclient.git
synced 2026-02-21 10:27:26 +08:00
refactor storage approach for proof trees due to pebble limitations
This commit is contained in:
Cassandra Heart
parent
7188f828a6
commit
1e63348a61
@ -24,8 +24,6 @@ replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go-
|
||||
|
||||
replace source.quilibrium.com/quilibrium/monorepo/node => ../node
|
||||
|
||||
replace github.com/cockroachdb/pebble => ../pebble
|
||||
|
||||
require (
|
||||
github.com/iden3/go-iden3-crypto v0.0.16
|
||||
github.com/multiformats/go-multiaddr v0.12.4
|
||||
@ -44,9 +42,10 @@ require (
|
||||
github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401 // indirect
|
||||
github.com/bwesterb/go-ristretto v1.2.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/pebble v0.0.0-20231210175920-b4d301aeb46a // indirect
|
||||
github.com/cockroachdb/pebble v1.1.4 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/consensys/gnark-crypto v0.5.3 // indirect
|
||||
@ -58,7 +57,7 @@ require (
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/getsentry/sentry-go v0.18.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/go-logr/logr v1.3.0 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
|
||||
@ -50,12 +50,14 @@ github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
|
||||
github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
|
||||
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
|
||||
github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8=
|
||||
github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
@ -104,8 +106,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
|
||||
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
|
||||
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
|
||||
github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
|
||||
@ -18,11 +18,22 @@ func init() {
|
||||
}
|
||||
|
||||
const (
|
||||
BranchNodes = 64
|
||||
BranchBits = 6 // log2(64)
|
||||
BranchMask = BranchNodes - 1
|
||||
NodesPerBranch = 64
|
||||
BranchBits = 6 // log2(64)
|
||||
BranchMask = NodesPerBranch - 1
|
||||
)
|
||||
|
||||
type VectorCommitmentTree interface {
|
||||
Commit(recalculate bool) []byte
|
||||
Delete(key []byte) error
|
||||
Get(key []byte) ([]byte, error)
|
||||
GetMetadata() (leafCount int, longestBranch int)
|
||||
GetSize() *big.Int
|
||||
Insert(key []byte, value []byte, hashTarget []byte, size *big.Int) error
|
||||
Prove(key []byte) [][]byte
|
||||
Verify(key []byte, proofs [][]byte) bool
|
||||
}
|
||||
|
||||
type VectorCommitmentNode interface {
|
||||
Commit(recalculate bool) []byte
|
||||
GetSize() *big.Int
|
||||
@ -38,7 +49,7 @@ type VectorCommitmentLeafNode struct {
|
||||
|
||||
type VectorCommitmentBranchNode struct {
|
||||
Prefix []int
|
||||
Children [BranchNodes]VectorCommitmentNode
|
||||
Children [NodesPerBranch]VectorCommitmentNode
|
||||
Commitment []byte
|
||||
Size *big.Int
|
||||
LeafCount int
|
||||
@ -175,12 +186,12 @@ func (n *VectorCommitmentBranchNode) Prove(index int) []byte {
|
||||
return rbls48581.ProveRaw(data, uint64(index), 64)
|
||||
}
|
||||
|
||||
type VectorCommitmentTree struct {
|
||||
type RawVectorCommitmentTree struct {
|
||||
Root VectorCommitmentNode
|
||||
}
|
||||
|
||||
// getNextNibble returns the next BranchBits bits from the key starting at pos
|
||||
func getNextNibble(key []byte, pos int) int {
|
||||
// GetNextNibble returns the next BranchBits bits from the key starting at pos
|
||||
func GetNextNibble(key []byte, pos int) int {
|
||||
startByte := pos / 8
|
||||
if startByte >= len(key) {
|
||||
return 0
|
||||
@ -208,13 +219,13 @@ func getNextNibble(key []byte, pos int) int {
|
||||
return result & BranchMask
|
||||
}
|
||||
|
||||
func getNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) {
|
||||
func GetNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) {
|
||||
var nibbles []int
|
||||
depth := startDepth
|
||||
|
||||
for {
|
||||
n1 := getNextNibble(key1, depth)
|
||||
n2 := getNextNibble(key2, depth)
|
||||
n1 := GetNextNibble(key1, depth)
|
||||
n2 := GetNextNibble(key2, depth)
|
||||
if n1 != n2 {
|
||||
return nibbles, depth
|
||||
}
|
||||
@ -224,7 +235,7 @@ func getNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) {
|
||||
}
|
||||
|
||||
// Insert adds or updates a key-value pair in the tree
|
||||
func (t *VectorCommitmentTree) Insert(
|
||||
func (t *RawVectorCommitmentTree) Insert(
|
||||
key, value, hashTarget []byte,
|
||||
size *big.Int,
|
||||
) error {
|
||||
@ -253,7 +264,7 @@ func (t *VectorCommitmentTree) Insert(
|
||||
}
|
||||
|
||||
// Get common prefix nibbles and divergence point
|
||||
sharedNibbles, divergeDepth := getNibblesUntilDiverge(n.Key, key, depth)
|
||||
sharedNibbles, divergeDepth := GetNibblesUntilDiverge(n.Key, key, depth)
|
||||
|
||||
// Create single branch node with shared prefix
|
||||
branch := &VectorCommitmentBranchNode{
|
||||
@ -264,8 +275,8 @@ func (t *VectorCommitmentTree) Insert(
|
||||
}
|
||||
|
||||
// Add both leaves at their final positions
|
||||
finalOldNibble := getNextNibble(n.Key, divergeDepth)
|
||||
finalNewNibble := getNextNibble(key, divergeDepth)
|
||||
finalOldNibble := GetNextNibble(n.Key, divergeDepth)
|
||||
finalNewNibble := GetNextNibble(key, divergeDepth)
|
||||
branch.Children[finalOldNibble] = n
|
||||
branch.Children[finalNewNibble] = &VectorCommitmentLeafNode{
|
||||
Key: key,
|
||||
@ -280,7 +291,7 @@ func (t *VectorCommitmentTree) Insert(
|
||||
if len(n.Prefix) > 0 {
|
||||
// Check if the new key matches the prefix
|
||||
for i, expectedNibble := range n.Prefix {
|
||||
actualNibble := getNextNibble(key, depth+i*BranchBits)
|
||||
actualNibble := GetNextNibble(key, depth+i*BranchBits)
|
||||
if actualNibble != expectedNibble {
|
||||
// Create new branch with shared prefix subset
|
||||
newBranch := &VectorCommitmentBranchNode{
|
||||
@ -303,7 +314,7 @@ func (t *VectorCommitmentTree) Insert(
|
||||
}
|
||||
|
||||
// Key matches prefix, continue with final nibble
|
||||
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
inserted := insert(
|
||||
n.Children[finalNibble],
|
||||
depth+len(n.Prefix)*BranchBits+BranchBits,
|
||||
@ -323,7 +334,7 @@ func (t *VectorCommitmentTree) Insert(
|
||||
return n
|
||||
} else {
|
||||
// Simple branch without prefix
|
||||
nibble := getNextNibble(key, depth)
|
||||
nibble := GetNextNibble(key, depth)
|
||||
inserted := insert(n.Children[nibble], depth+BranchBits)
|
||||
n.Children[nibble] = inserted
|
||||
n.Commitment = nil
|
||||
@ -348,7 +359,7 @@ func (t *VectorCommitmentTree) Insert(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
|
||||
func (t *RawVectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
|
||||
if len(key) == 0 {
|
||||
return false
|
||||
}
|
||||
@ -373,13 +384,13 @@ func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
|
||||
case *VectorCommitmentBranchNode:
|
||||
// Check prefix match
|
||||
for i, expectedNibble := range n.Prefix {
|
||||
if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
if GetNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Get final nibble after prefix
|
||||
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
|
||||
if !n.Verify(finalNibble, proofs[0]) {
|
||||
return false
|
||||
@ -394,7 +405,7 @@ func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
|
||||
return verify(t.Root, proofs, 0)
|
||||
}
|
||||
|
||||
func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
|
||||
func (t *RawVectorCommitmentTree) Prove(key []byte) [][]byte {
|
||||
if len(key) == 0 {
|
||||
return nil
|
||||
}
|
||||
@ -415,13 +426,13 @@ func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
|
||||
case *VectorCommitmentBranchNode:
|
||||
// Check prefix match
|
||||
for i, expectedNibble := range n.Prefix {
|
||||
if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
if GetNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Get final nibble after prefix
|
||||
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
|
||||
proofs := [][]byte{n.Prove(finalNibble)}
|
||||
|
||||
@ -435,7 +446,7 @@ func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
|
||||
}
|
||||
|
||||
// Get retrieves a value from the tree by key
|
||||
func (t *VectorCommitmentTree) Get(key []byte) ([]byte, error) {
|
||||
func (t *RawVectorCommitmentTree) Get(key []byte) ([]byte, error) {
|
||||
if len(key) == 0 {
|
||||
return nil, errors.New("empty key not allowed")
|
||||
}
|
||||
@ -456,12 +467,12 @@ func (t *VectorCommitmentTree) Get(key []byte) ([]byte, error) {
|
||||
case *VectorCommitmentBranchNode:
|
||||
// Check prefix match
|
||||
for i, expectedNibble := range n.Prefix {
|
||||
if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
if GetNextNibble(key, depth+i*BranchBits) != expectedNibble {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Get final nibble after prefix
|
||||
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
return get(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits)
|
||||
}
|
||||
|
||||
@ -476,7 +487,7 @@ func (t *VectorCommitmentTree) Get(key []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
// Delete removes a key-value pair from the tree
|
||||
func (t *VectorCommitmentTree) Delete(key []byte) error {
|
||||
func (t *RawVectorCommitmentTree) Delete(key []byte) error {
|
||||
if len(key) == 0 {
|
||||
return errors.New("empty key not allowed")
|
||||
}
|
||||
@ -497,13 +508,13 @@ func (t *VectorCommitmentTree) Delete(key []byte) error {
|
||||
|
||||
case *VectorCommitmentBranchNode:
|
||||
for i, expectedNibble := range n.Prefix {
|
||||
currentNibble := getNextNibble(key, depth+i*BranchBits)
|
||||
currentNibble := GetNextNibble(key, depth+i*BranchBits)
|
||||
if currentNibble != expectedNibble {
|
||||
return big.NewInt(0), n
|
||||
}
|
||||
}
|
||||
|
||||
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
finalNibble := GetNextNibble(key, depth+len(n.Prefix)*BranchBits)
|
||||
var size *big.Int
|
||||
size, n.Children[finalNibble] =
|
||||
remove(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits)
|
||||
@ -568,7 +579,10 @@ func (t *VectorCommitmentTree) Delete(key []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *VectorCommitmentTree) GetMetadata() (leafCount int, longestBranch int) {
|
||||
func (t *RawVectorCommitmentTree) GetMetadata() (
|
||||
leafCount int,
|
||||
longestBranch int,
|
||||
) {
|
||||
switch root := t.Root.(type) {
|
||||
case nil:
|
||||
return 0, 0
|
||||
@ -581,14 +595,14 @@ func (t *VectorCommitmentTree) GetMetadata() (leafCount int, longestBranch int)
|
||||
}
|
||||
|
||||
// Commit returns the root of the tree
|
||||
func (t *VectorCommitmentTree) Commit(recalculate bool) []byte {
|
||||
func (t *RawVectorCommitmentTree) Commit(recalculate bool) []byte {
|
||||
if t.Root == nil {
|
||||
return make([]byte, 64)
|
||||
}
|
||||
return t.Root.Commit(recalculate)
|
||||
}
|
||||
|
||||
func (t *VectorCommitmentTree) GetSize() *big.Int {
|
||||
func (t *RawVectorCommitmentTree) GetSize() *big.Int {
|
||||
return t.Root.GetSize()
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
func BenchmarkVectorCommitmentTreeInsert(b *testing.B) {
|
||||
tree := &VectorCommitmentTree{}
|
||||
tree := &RawVectorCommitmentTree{}
|
||||
addresses := [][]byte{}
|
||||
|
||||
for i := range b.N {
|
||||
@ -26,7 +26,7 @@ func BenchmarkVectorCommitmentTreeInsert(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkVectorCommitmentTreeCommit(b *testing.B) {
|
||||
tree := &VectorCommitmentTree{}
|
||||
tree := &RawVectorCommitmentTree{}
|
||||
addresses := [][]byte{}
|
||||
|
||||
for i := range b.N {
|
||||
@ -42,7 +42,7 @@ func BenchmarkVectorCommitmentTreeCommit(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkVectorCommitmentTreeProve(b *testing.B) {
|
||||
tree := &VectorCommitmentTree{}
|
||||
tree := &RawVectorCommitmentTree{}
|
||||
addresses := [][]byte{}
|
||||
|
||||
for i := range b.N {
|
||||
@ -58,7 +58,7 @@ func BenchmarkVectorCommitmentTreeProve(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkVectorCommitmentTreeVerify(b *testing.B) {
|
||||
tree := &VectorCommitmentTree{}
|
||||
tree := &RawVectorCommitmentTree{}
|
||||
addresses := [][]byte{}
|
||||
|
||||
for i := range b.N {
|
||||
@ -78,7 +78,7 @@ func BenchmarkVectorCommitmentTreeVerify(b *testing.B) {
|
||||
|
||||
func TestVectorCommitmentTrees(t *testing.T) {
|
||||
bls48581.Init()
|
||||
tree := &VectorCommitmentTree{}
|
||||
tree := &RawVectorCommitmentTree{}
|
||||
|
||||
// Test single insert
|
||||
err := tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1))
|
||||
@ -106,7 +106,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
t.Error("Expected error for empty key, got none")
|
||||
}
|
||||
|
||||
tree = &VectorCommitmentTree{}
|
||||
tree = &RawVectorCommitmentTree{}
|
||||
|
||||
// Test get on empty tree
|
||||
_, err = tree.Get([]byte("nonexistent"))
|
||||
@ -130,7 +130,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
t.Error("Expected error for empty key, got none")
|
||||
}
|
||||
|
||||
tree = &VectorCommitmentTree{}
|
||||
tree = &RawVectorCommitmentTree{}
|
||||
|
||||
// Test delete on empty tree
|
||||
err = tree.Delete([]byte("nonexistent"))
|
||||
@ -157,7 +157,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
t.Error("Expected error for empty key, got none")
|
||||
}
|
||||
|
||||
tree = &VectorCommitmentTree{}
|
||||
tree = &RawVectorCommitmentTree{}
|
||||
|
||||
// Insert keys that share common prefix
|
||||
keys := []string{
|
||||
@ -212,7 +212,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
tree = &VectorCommitmentTree{}
|
||||
tree = &RawVectorCommitmentTree{}
|
||||
|
||||
// Empty tree should be empty
|
||||
emptyRoot := tree.Root
|
||||
@ -244,8 +244,8 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
t.Error("Root hash should match empty tree after deleting all entries")
|
||||
}
|
||||
|
||||
tree = &VectorCommitmentTree{}
|
||||
cmptree := &VectorCommitmentTree{}
|
||||
tree = &RawVectorCommitmentTree{}
|
||||
cmptree := &RawVectorCommitmentTree{}
|
||||
|
||||
addresses := [][]byte{}
|
||||
|
||||
@ -332,7 +332,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
|
||||
t.Errorf("invalid tree size: %s", tree.GetSize().String())
|
||||
}
|
||||
|
||||
cmptree = &VectorCommitmentTree{}
|
||||
cmptree = &RawVectorCommitmentTree{}
|
||||
|
||||
for i := 0; i < 10000; i++ {
|
||||
cmptree.Insert(kept[i], kept[i], nil, big.NewInt(1))
|
||||
|
||||
@ -6,7 +6,7 @@ import (
|
||||
)
|
||||
|
||||
// CompareTreesAtHeight compares two vector commitment trees at each level
|
||||
func CompareTreesAtHeight(tree1, tree2 *VectorCommitmentTree) [][]ComparisonResult {
|
||||
func CompareTreesAtHeight(tree1, tree2 *RawVectorCommitmentTree) [][]ComparisonResult {
|
||||
if tree1 == nil || tree2 == nil {
|
||||
return nil
|
||||
}
|
||||
@ -104,7 +104,7 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren
|
||||
|
||||
// If we're still below target height after prefix, traverse children
|
||||
if nextHeight < targetHeight {
|
||||
for i := 0; i < BranchNodes; i++ {
|
||||
for i := 0; i < NodesPerBranch; i++ {
|
||||
childResults := compareLevelCommits(n1.Children[i], n2.Children[i], targetHeight, nextHeight+1)
|
||||
results = append(results, childResults...)
|
||||
}
|
||||
@ -115,7 +115,7 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren
|
||||
}
|
||||
|
||||
// TraverseAndCompare provides a channel-based iterator for comparing trees
|
||||
func TraverseAndCompare(tree1, tree2 *VectorCommitmentTree) chan ComparisonResult {
|
||||
func TraverseAndCompare(tree1, tree2 *RawVectorCommitmentTree) chan ComparisonResult {
|
||||
resultChan := make(chan ComparisonResult)
|
||||
|
||||
go func() {
|
||||
@ -150,7 +150,7 @@ type LeafDifference struct {
|
||||
}
|
||||
|
||||
// CompareLeaves returns all leaves that differ between the two trees
|
||||
func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference {
|
||||
func CompareLeaves(tree1, tree2 *RawVectorCommitmentTree) []LeafDifference {
|
||||
// Get all leaves from both trees
|
||||
leaves1 := GetAllLeaves(tree1.Root)
|
||||
leaves2 := GetAllLeaves(tree2.Root)
|
||||
@ -231,8 +231,8 @@ func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode {
|
||||
|
||||
func ExampleComparison() {
|
||||
// Create and populate two trees
|
||||
tree1 := &VectorCommitmentTree{}
|
||||
tree2 := &VectorCommitmentTree{}
|
||||
tree1 := &RawVectorCommitmentTree{}
|
||||
tree2 := &RawVectorCommitmentTree{}
|
||||
|
||||
// Compare trees using channel-based iterator
|
||||
for result := range TraverseAndCompare(tree1, tree2) {
|
||||
|
||||
@ -148,7 +148,18 @@ func NewTokenExecutionEngine(
|
||||
var inclusionProof *qcrypto.InclusionAggregateProof
|
||||
var proverKeys [][]byte
|
||||
var peerSeniority map[string]uint64
|
||||
hg := hypergraph.NewHypergraph()
|
||||
hg := hypergraph.NewHypergraph(
|
||||
func(
|
||||
shardKey hypergraph.ShardKey,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
) qcrypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
hypergraphStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
},
|
||||
)
|
||||
mpcithVerEnc := qcrypto.NewMPCitHVerifiableEncryptor(
|
||||
runtime.NumCPU(),
|
||||
)
|
||||
@ -484,7 +495,7 @@ func (e *TokenExecutionEngine) addBatchToHypergraph(batchKey [][]byte, batchValu
|
||||
var wg sync.WaitGroup
|
||||
throttle := make(chan struct{}, runtime.NumCPU())
|
||||
batchCompressed := make([]hypergraph.Vertex, len(batchKey))
|
||||
batchTrees := make([]*qcrypto.VectorCommitmentTree, len(batchKey))
|
||||
batchTrees := make([]*qcrypto.RawVectorCommitmentTree, len(batchKey))
|
||||
txn, err := e.hypergraphStore.NewTransaction(false)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@ -639,7 +650,18 @@ func (e *TokenExecutionEngine) hyperSync() {
|
||||
|
||||
func (e *TokenExecutionEngine) rebuildHypergraph() {
|
||||
e.logger.Info("rebuilding hypergraph")
|
||||
e.hypergraph = hypergraph.NewHypergraph()
|
||||
e.hypergraph = hypergraph.NewHypergraph(
|
||||
func(
|
||||
shardKey hypergraph.ShardKey,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
) qcrypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
e.hypergraphStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
},
|
||||
)
|
||||
if e.engineConfig.RebuildStart == "" {
|
||||
e.engineConfig.RebuildStart = "0000000000000000000000000000000000000000000000000000000000000000"
|
||||
}
|
||||
|
||||
11
node/go.mod
11
node/go.mod
@ -23,10 +23,8 @@ replace github.com/libp2p/go-libp2p-kad-dht => ../go-libp2p-kad-dht
|
||||
|
||||
replace source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub => ../go-libp2p-blossomsub
|
||||
|
||||
replace github.com/cockroachdb/pebble => ../pebble
|
||||
|
||||
require (
|
||||
github.com/cockroachdb/pebble v0.0.0-20231210175920-b4d301aeb46a
|
||||
github.com/cockroachdb/pebble v1.1.4
|
||||
github.com/deiu/rdf2go v0.0.0-20240619132609-81222e324bb9
|
||||
github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1
|
||||
github.com/klauspost/reedsolomon v1.12.4
|
||||
@ -45,8 +43,8 @@ require (
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.0.0-rc.1 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/deiu/gon3 v0.0.0-20230411081920-f0f8f879f597 // indirect
|
||||
github.com/google/subcommands v1.0.1 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
|
||||
github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect
|
||||
github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 // indirect
|
||||
@ -80,14 +78,14 @@ require (
|
||||
github.com/bwesterb/go-ristretto v1.2.3 // indirect
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/lipgloss v0.9.1
|
||||
github.com/cockroachdb/errors v1.11.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/consensys/gnark-crypto v0.5.3 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/getsentry/sentry-go v0.18.0 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0
|
||||
@ -112,7 +110,6 @@ require (
|
||||
require (
|
||||
github.com/benbjohnson/clock v1.3.5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cbergoon/merkletree v0.2.0
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.9
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
|
||||
17
node/go.sum
17
node/go.sum
@ -42,8 +42,6 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/cbergoon/merkletree v0.2.0 h1:Bttqr3OuoiZEo4ed1L7fTasHka9II+BF9fhBfbNEEoQ=
|
||||
github.com/cbergoon/merkletree v0.2.0/go.mod h1:5c15eckUgiucMGDOCanvalj/yJnD+KAZj1qyJtRW5aM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@ -58,12 +56,14 @@ github.com/cloudflare/circl v1.3.9/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZ
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
|
||||
github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
|
||||
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
|
||||
github.com/cockroachdb/pebble v1.1.4 h1:5II1uEP4MyHLDnsrbv/EZ36arcb9Mxg3n+owhZ3GrG8=
|
||||
github.com/cockroachdb/pebble v1.1.4/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
@ -117,8 +117,8 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD
|
||||
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
|
||||
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
|
||||
github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
@ -183,7 +183,6 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
|
||||
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo=
|
||||
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k=
|
||||
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/crypto"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/p2p"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
|
||||
)
|
||||
|
||||
type AtomType string
|
||||
@ -72,7 +73,7 @@ type hyperedge struct {
|
||||
appAddress [32]byte
|
||||
dataAddress [32]byte
|
||||
extrinsics map[[64]byte]Atom
|
||||
extTree *crypto.VectorCommitmentTree
|
||||
extTree *crypto.RawVectorCommitmentTree
|
||||
}
|
||||
|
||||
var _ Vertex = (*vertex)(nil)
|
||||
@ -89,8 +90,8 @@ type Atom interface {
|
||||
Commit() []byte
|
||||
}
|
||||
|
||||
func EncryptedToVertexTree(encrypted []Encrypted) *crypto.VectorCommitmentTree {
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
func EncryptedToVertexTree(encrypted []Encrypted) *crypto.RawVectorCommitmentTree {
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range encrypted {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -118,7 +119,7 @@ func AtomFromBytes(data []byte) Atom {
|
||||
size: new(big.Int).SetBytes(data[len(data)-32:]),
|
||||
}
|
||||
} else {
|
||||
tree := &crypto.VectorCommitmentTree{}
|
||||
tree := &crypto.RawVectorCommitmentTree{}
|
||||
var b bytes.Buffer
|
||||
b.Write(data[65:])
|
||||
dec := gob.NewDecoder(&b)
|
||||
@ -162,7 +163,7 @@ func NewHyperedge(
|
||||
appAddress: appAddress,
|
||||
dataAddress: dataAddress,
|
||||
extrinsics: make(map[[64]byte]Atom),
|
||||
extTree: &crypto.VectorCommitmentTree{},
|
||||
extTree: &crypto.RawVectorCommitmentTree{},
|
||||
}
|
||||
}
|
||||
|
||||
@ -330,48 +331,38 @@ type IdSet struct {
|
||||
dirty bool
|
||||
atomType AtomType
|
||||
atoms map[[64]byte]Atom
|
||||
tree *crypto.VectorCommitmentTree
|
||||
tree crypto.VectorCommitmentTree
|
||||
}
|
||||
|
||||
func NewIdSet(atomType AtomType) *IdSet {
|
||||
func NewIdSet(
|
||||
shardKey ShardKey,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
tree crypto.VectorCommitmentTree,
|
||||
) *IdSet {
|
||||
atomType := VertexAtomType
|
||||
switch phaseSet {
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS:
|
||||
fallthrough
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES:
|
||||
atomType = VertexAtomType
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS:
|
||||
fallthrough
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES:
|
||||
atomType = HyperedgeAtomType
|
||||
}
|
||||
|
||||
return &IdSet{
|
||||
dirty: false,
|
||||
atomType: atomType,
|
||||
atoms: make(map[[64]byte]Atom),
|
||||
tree: &crypto.VectorCommitmentTree{},
|
||||
tree: tree,
|
||||
}
|
||||
}
|
||||
|
||||
func (set *IdSet) FromBytes(treeData []byte) error {
|
||||
set.tree = &crypto.VectorCommitmentTree{}
|
||||
var b bytes.Buffer
|
||||
b.Write(treeData)
|
||||
dec := gob.NewDecoder(&b)
|
||||
if err := dec.Decode(set.tree); err != nil {
|
||||
return errors.Wrap(err, "load set")
|
||||
}
|
||||
|
||||
for _, leaf := range crypto.GetAllLeaves(set.tree.Root) {
|
||||
set.atoms[[64]byte(leaf.Key)] = AtomFromBytes(leaf.Value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (set *IdSet) IsDirty() bool {
|
||||
return set.dirty
|
||||
}
|
||||
|
||||
func (set *IdSet) ToBytes() []byte {
|
||||
var buf bytes.Buffer
|
||||
enc := gob.NewEncoder(&buf)
|
||||
if err := enc.Encode(set.tree); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (set *IdSet) Add(atom Atom) error {
|
||||
if atom.GetAtomType() != set.atomType {
|
||||
return ErrInvalidAtomType
|
||||
@ -412,21 +403,31 @@ func (set *IdSet) Has(key [64]byte) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (set *IdSet) GetTree() *crypto.VectorCommitmentTree {
|
||||
func (set *IdSet) GetTree() crypto.VectorCommitmentTree {
|
||||
return set.tree
|
||||
}
|
||||
|
||||
type Hypergraph struct {
|
||||
size *big.Int
|
||||
size *big.Int
|
||||
treeConstructor func(
|
||||
shardKey ShardKey,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
) crypto.VectorCommitmentTree
|
||||
vertexAdds map[ShardKey]*IdSet
|
||||
vertexRemoves map[ShardKey]*IdSet
|
||||
hyperedgeAdds map[ShardKey]*IdSet
|
||||
hyperedgeRemoves map[ShardKey]*IdSet
|
||||
}
|
||||
|
||||
func NewHypergraph() *Hypergraph {
|
||||
func NewHypergraph(
|
||||
treeConstructor func(
|
||||
shardKey ShardKey,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
) crypto.VectorCommitmentTree,
|
||||
) *Hypergraph {
|
||||
return &Hypergraph{
|
||||
size: big.NewInt(0),
|
||||
treeConstructor: treeConstructor,
|
||||
vertexAdds: make(map[ShardKey]*IdSet),
|
||||
vertexRemoves: make(map[ShardKey]*IdSet),
|
||||
hyperedgeAdds: make(map[ShardKey]*IdSet),
|
||||
@ -467,36 +468,26 @@ func (hg *Hypergraph) Commit() [][]byte {
|
||||
return commits
|
||||
}
|
||||
|
||||
func (hg *Hypergraph) ImportFromBytes(
|
||||
atomType AtomType,
|
||||
phaseType PhaseType,
|
||||
func (hg *Hypergraph) SetIdSet(
|
||||
shardKey ShardKey,
|
||||
data []byte,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
tree crypto.VectorCommitmentTree,
|
||||
) error {
|
||||
set := NewIdSet(atomType)
|
||||
if err := set.FromBytes(data); err != nil {
|
||||
return errors.Wrap(err, "import from bytes")
|
||||
}
|
||||
set := NewIdSet(shardKey, phaseSet, tree)
|
||||
|
||||
switch atomType {
|
||||
case VertexAtomType:
|
||||
switch phaseType {
|
||||
case AddsPhaseType:
|
||||
hg.size.Add(hg.size, set.GetSize())
|
||||
hg.vertexAdds[shardKey] = set
|
||||
case RemovesPhaseType:
|
||||
hg.size.Sub(hg.size, set.GetSize())
|
||||
hg.vertexRemoves[shardKey] = set
|
||||
}
|
||||
case HyperedgeAtomType:
|
||||
switch phaseType {
|
||||
case AddsPhaseType:
|
||||
hg.size.Add(hg.size, set.GetSize())
|
||||
hg.hyperedgeAdds[shardKey] = set
|
||||
case RemovesPhaseType:
|
||||
hg.size.Sub(hg.size, set.GetSize())
|
||||
hg.hyperedgeRemoves[shardKey] = set
|
||||
}
|
||||
switch phaseSet {
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS:
|
||||
hg.size.Add(hg.size, set.GetSize())
|
||||
hg.vertexAdds[shardKey] = set
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES:
|
||||
hg.size.Sub(hg.size, set.GetSize())
|
||||
hg.vertexRemoves[shardKey] = set
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS:
|
||||
hg.size.Add(hg.size, set.GetSize())
|
||||
hg.hyperedgeAdds[shardKey] = set
|
||||
case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES:
|
||||
hg.size.Sub(hg.size, set.GetSize())
|
||||
hg.hyperedgeRemoves[shardKey] = set
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -507,18 +498,32 @@ func (hg *Hypergraph) GetSize() *big.Int {
|
||||
}
|
||||
|
||||
func (hg *Hypergraph) getOrCreateIdSet(
|
||||
shardAddr ShardKey,
|
||||
shardKey ShardKey,
|
||||
addMap map[ShardKey]*IdSet,
|
||||
removeMap map[ShardKey]*IdSet,
|
||||
atomType AtomType,
|
||||
) (*IdSet, *IdSet) {
|
||||
if _, ok := addMap[shardAddr]; !ok {
|
||||
addMap[shardAddr] = NewIdSet(atomType)
|
||||
if _, ok := addMap[shardKey]; !ok {
|
||||
phaseSet := protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS
|
||||
if atomType == HyperedgeAtomType {
|
||||
phaseSet = protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_ADDS
|
||||
}
|
||||
addMap[shardKey] = NewIdSet(shardKey, phaseSet, hg.treeConstructor(
|
||||
shardKey,
|
||||
phaseSet,
|
||||
))
|
||||
}
|
||||
if _, ok := removeMap[shardAddr]; !ok {
|
||||
removeMap[shardAddr] = NewIdSet(atomType)
|
||||
if _, ok := removeMap[shardKey]; !ok {
|
||||
phaseSet := protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_REMOVES
|
||||
if atomType == HyperedgeAtomType {
|
||||
phaseSet = protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_HYPEREDGE_REMOVES
|
||||
}
|
||||
removeMap[shardKey] = NewIdSet(shardKey, phaseSet, hg.treeConstructor(
|
||||
shardKey,
|
||||
phaseSet,
|
||||
))
|
||||
}
|
||||
return addMap[shardAddr], removeMap[shardAddr]
|
||||
return addMap[shardKey], removeMap[shardKey]
|
||||
}
|
||||
|
||||
func (hg *Hypergraph) AddVertex(v Vertex) error {
|
||||
@ -687,37 +692,3 @@ func (hg *Hypergraph) Within(a, h Atom) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (hg *Hypergraph) GetReconciledVertexSetForShard(
|
||||
shardKey ShardKey,
|
||||
) *IdSet {
|
||||
vertices := NewIdSet(VertexAtomType)
|
||||
|
||||
if addSet, ok := hg.vertexAdds[shardKey]; ok {
|
||||
removeSet := hg.vertexRemoves[shardKey]
|
||||
for id, v := range addSet.atoms {
|
||||
if !removeSet.Has(id) {
|
||||
vertices.Add(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return vertices
|
||||
}
|
||||
|
||||
func (hg *Hypergraph) GetReconciledHyperedgeSetForShard(
|
||||
shardKey ShardKey,
|
||||
) *IdSet {
|
||||
hyperedges := NewIdSet(HyperedgeAtomType)
|
||||
|
||||
if addSet, ok := hg.hyperedgeAdds[shardKey]; ok {
|
||||
removeSet := hg.hyperedgeRemoves[shardKey]
|
||||
for _, h := range addSet.atoms {
|
||||
if !removeSet.Has(h.GetID()) {
|
||||
hyperedges.Add(h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return hyperedges
|
||||
}
|
||||
|
||||
@ -10,8 +10,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/cloudflare/circl/sign/ed448"
|
||||
"go.uber.org/zap"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/crypto"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/store"
|
||||
)
|
||||
|
||||
type Operation struct {
|
||||
@ -22,13 +25,13 @@ type Operation struct {
|
||||
|
||||
func TestConvergence(t *testing.T) {
|
||||
numParties := 4
|
||||
numOperations := 100000
|
||||
numOperations := 10000
|
||||
enc := crypto.NewMPCitHVerifiableEncryptor(1)
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
vertices := make([]application.Vertex, numOperations)
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -77,9 +80,19 @@ func TestConvergence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
inmem := store.NewInMemKVDB()
|
||||
logger, _ := zap.NewProduction()
|
||||
hgStore := store.NewPebbleHypergraphStore(inmem, logger)
|
||||
|
||||
crdts := make([]*application.Hypergraph, numParties)
|
||||
for i := 0; i < numParties; i++ {
|
||||
crdts[i] = application.NewHypergraph()
|
||||
crdts[i] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
hgStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < numParties; i++ {
|
||||
|
||||
@ -11,10 +11,13 @@ import (
|
||||
"github.com/cloudflare/circl/sign/ed448"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/crypto"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application"
|
||||
"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
|
||||
)
|
||||
|
||||
func TestHypergraph(t *testing.T) {
|
||||
hg := application.NewHypergraph()
|
||||
hg := application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree {
|
||||
return &crypto.RawVectorCommitmentTree{}
|
||||
})
|
||||
|
||||
// Test vertex operations
|
||||
t.Run("Vertex Operations", func(t *testing.T) {
|
||||
@ -22,7 +25,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -69,7 +72,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -113,7 +116,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -151,7 +154,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -186,7 +189,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -222,7 +225,7 @@ func TestHypergraph(t *testing.T) {
|
||||
pub, _, _ := ed448.GenerateKey(crand.Reader)
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
|
||||
@ -43,7 +43,7 @@ func NewHypergraphComparisonServer(
|
||||
func sendLeafData(
|
||||
stream protobufs.HypergraphComparisonService_HyperStreamClient,
|
||||
hypergraphStore store.HypergraphStore,
|
||||
localTree *crypto.VectorCommitmentTree,
|
||||
localTree *crypto.RawVectorCommitmentTree,
|
||||
path []int32,
|
||||
metadataOnly bool,
|
||||
) error {
|
||||
@ -161,7 +161,7 @@ func getNodeAtPath(
|
||||
// getBranchInfoFromTree looks up the node at the given path in the local tree,
|
||||
// computes its commitment, and (if it is a branch) collects its immediate
|
||||
// children's commitments.
|
||||
func getBranchInfoFromTree(tree *crypto.VectorCommitmentTree, path []int32) (
|
||||
func getBranchInfoFromTree(tree *crypto.RawVectorCommitmentTree, path []int32) (
|
||||
*protobufs.HypergraphComparisonResponse,
|
||||
error,
|
||||
) {
|
||||
@ -207,7 +207,7 @@ func isLeaf(info *protobufs.HypergraphComparisonResponse) bool {
|
||||
func sendLeafDataServer(
|
||||
stream protobufs.HypergraphComparisonService_HyperStreamServer,
|
||||
hypergraphStore store.HypergraphStore,
|
||||
localTree *crypto.VectorCommitmentTree,
|
||||
localTree *crypto.RawVectorCommitmentTree,
|
||||
path []int32,
|
||||
metadataOnly bool,
|
||||
) error {
|
||||
@ -309,7 +309,10 @@ func syncTreeBidirectionallyServer(
|
||||
|
||||
// Send our root branch info.
|
||||
rootPath := []int32{}
|
||||
rootInfo, err := getBranchInfoFromTree(idSet.GetTree(), rootPath)
|
||||
rootInfo, err := getBranchInfoFromTree(
|
||||
idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
rootPath,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -374,7 +377,7 @@ func syncTreeBidirectionallyServer(
|
||||
zap.String("path", hex.EncodeToString(packNibbles(remoteInfo.Path))),
|
||||
)
|
||||
localInfo, err := getBranchInfoFromTree(
|
||||
idSet.GetTree(),
|
||||
idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
remoteInfo.Path,
|
||||
)
|
||||
if err != nil {
|
||||
@ -422,7 +425,7 @@ func syncTreeBidirectionallyServer(
|
||||
if err := sendLeafDataServer(
|
||||
stream,
|
||||
localHypergraphStore,
|
||||
idSet.GetTree(),
|
||||
idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
remoteInfo.Path,
|
||||
metadataOnly,
|
||||
); err != nil {
|
||||
@ -477,7 +480,7 @@ func syncTreeBidirectionallyServer(
|
||||
if err := sendLeafDataServer(
|
||||
stream,
|
||||
localHypergraphStore,
|
||||
idSet.GetTree(),
|
||||
idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
queryPath,
|
||||
metadataOnly,
|
||||
); err != nil {
|
||||
@ -491,7 +494,10 @@ func syncTreeBidirectionallyServer(
|
||||
hex.EncodeToString(packNibbles(queryPath)),
|
||||
),
|
||||
)
|
||||
branchInfo, err := getBranchInfoFromTree(idSet.GetTree(), queryPath)
|
||||
branchInfo, err := getBranchInfoFromTree(
|
||||
idSet.GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
queryPath,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@ -518,7 +524,7 @@ func syncTreeBidirectionallyServer(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tree := &crypto.VectorCommitmentTree{}
|
||||
tree := &crypto.RawVectorCommitmentTree{}
|
||||
var b bytes.Buffer
|
||||
b.Write(remoteUpdate.UnderlyingData)
|
||||
|
||||
@ -570,7 +576,7 @@ func SyncTreeBidirectionally(
|
||||
shardKey []byte,
|
||||
phaseSet protobufs.HypergraphPhaseSet,
|
||||
hypergraphStore store.HypergraphStore,
|
||||
localTree *crypto.VectorCommitmentTree,
|
||||
localTree crypto.VectorCommitmentTree,
|
||||
metadataOnly bool,
|
||||
) error {
|
||||
logger.Info(
|
||||
@ -593,7 +599,10 @@ func SyncTreeBidirectionally(
|
||||
}
|
||||
|
||||
rootPath := []int32{}
|
||||
rootInfo, err := getBranchInfoFromTree(localTree, rootPath)
|
||||
rootInfo, err := getBranchInfoFromTree(
|
||||
localTree.(*store.PersistentVectorTree).GetInternalTree(),
|
||||
rootPath,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -653,7 +662,10 @@ func SyncTreeBidirectionally(
|
||||
"handling response",
|
||||
zap.String("path", hex.EncodeToString(packNibbles(remoteInfo.Path))),
|
||||
)
|
||||
localInfo, err := getBranchInfoFromTree(localTree, remoteInfo.Path)
|
||||
localInfo, err := getBranchInfoFromTree(
|
||||
localTree.(*store.PersistentVectorTree).GetInternalTree(),
|
||||
remoteInfo.Path,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info(
|
||||
"requesting missing node",
|
||||
@ -695,7 +707,7 @@ func SyncTreeBidirectionally(
|
||||
if err := sendLeafData(
|
||||
stream,
|
||||
hypergraphStore,
|
||||
localTree,
|
||||
localTree.(*store.PersistentVectorTree).GetInternalTree(),
|
||||
remoteInfo.Path,
|
||||
metadataOnly,
|
||||
); err != nil {
|
||||
@ -750,7 +762,7 @@ func SyncTreeBidirectionally(
|
||||
if err := sendLeafData(
|
||||
stream,
|
||||
hypergraphStore,
|
||||
localTree,
|
||||
localTree.(*store.PersistentVectorTree).GetInternalTree(),
|
||||
queryPath,
|
||||
metadataOnly,
|
||||
); err != nil {
|
||||
@ -764,7 +776,10 @@ func SyncTreeBidirectionally(
|
||||
hex.EncodeToString(packNibbles(queryPath)),
|
||||
),
|
||||
)
|
||||
branchInfo, err := getBranchInfoFromTree(localTree, queryPath)
|
||||
branchInfo, err := getBranchInfoFromTree(
|
||||
localTree.(*store.PersistentVectorTree).GetInternalTree(),
|
||||
queryPath,
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@ -792,7 +807,7 @@ func SyncTreeBidirectionally(
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tree := &crypto.VectorCommitmentTree{}
|
||||
tree := &crypto.RawVectorCommitmentTree{}
|
||||
var b bytes.Buffer
|
||||
b.Write(remoteUpdate.UnderlyingData)
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/cloudflare/circl/sign/ed448"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
@ -37,7 +38,7 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
data := enc.Encrypt(make([]byte, 20), pub)
|
||||
verenc := data[0].Compress()
|
||||
vertices := make([]application.Vertex, numOperations)
|
||||
dataTree := &crypto.VectorCommitmentTree{}
|
||||
dataTree := &crypto.RawVectorCommitmentTree{}
|
||||
for _, d := range []application.Encrypted{verenc} {
|
||||
dataBytes := d.ToBytes()
|
||||
id := sha512.Sum512(dataBytes)
|
||||
@ -88,15 +89,35 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
clientKvdb := store.NewInMemKVDB()
|
||||
serverKvdb := store.NewInMemKVDB()
|
||||
clientKvdb := store.NewInMemKVDB()
|
||||
controlKvdb := store.NewInMemKVDB()
|
||||
logger, _ := zap.NewProduction()
|
||||
clientHypergraphStore := store.NewPebbleHypergraphStore(clientKvdb, logger)
|
||||
serverHypergraphStore := store.NewPebbleHypergraphStore(serverKvdb, logger)
|
||||
clientHypergraphStore := store.NewPebbleHypergraphStore(clientKvdb, logger)
|
||||
controlHypergraphStore := store.NewPebbleHypergraphStore(controlKvdb, logger)
|
||||
crdts := make([]*application.Hypergraph, numParties)
|
||||
for i := 0; i < numParties; i++ {
|
||||
crdts[i] = application.NewHypergraph()
|
||||
}
|
||||
crdts[0] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
serverHypergraphStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
})
|
||||
crdts[1] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
clientHypergraphStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
})
|
||||
crdts[2] = application.NewHypergraph(func(shardKey application.ShardKey, phaseSet protobufs.HypergraphPhaseSet) crypto.VectorCommitmentTree {
|
||||
return store.NewPersistentVectorTree(
|
||||
controlHypergraphStore,
|
||||
shardKey,
|
||||
phaseSet,
|
||||
)
|
||||
})
|
||||
|
||||
txn, _ := serverHypergraphStore.NewTransaction(false)
|
||||
for _, op := range operations1[:5000] {
|
||||
@ -114,7 +135,7 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
txn.Commit()
|
||||
for _, op := range operations2[:500] {
|
||||
for _, op := range operations2[:5000] {
|
||||
switch op.Type {
|
||||
case "AddVertex":
|
||||
crdts[0].AddVertex(op.Vertex)
|
||||
@ -143,7 +164,7 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
txn.Commit()
|
||||
for _, op := range operations2[500:] {
|
||||
for _, op := range operations2[5000:] {
|
||||
switch op.Type {
|
||||
case "AddVertex":
|
||||
crdts[1].AddVertex(op.Vertex)
|
||||
@ -184,7 +205,31 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
crdts[0].Commit()
|
||||
crdts[1].Commit()
|
||||
crdts[2].Commit()
|
||||
txn, _ = serverHypergraphStore.NewTransaction(false)
|
||||
serverHypergraphStore.SaveHypergraph(txn, crdts[0])
|
||||
txn.Commit()
|
||||
txn, _ = clientHypergraphStore.NewTransaction(false)
|
||||
clientHypergraphStore.SaveHypergraph(txn, crdts[1])
|
||||
txn.Commit()
|
||||
txn, _ = controlHypergraphStore.NewTransaction(false)
|
||||
controlHypergraphStore.SaveHypergraph(txn, crdts[2])
|
||||
txn.Commit()
|
||||
var err error
|
||||
eval0, err := serverHypergraphStore.LoadHypergraph()
|
||||
assert.NoError(t, err)
|
||||
eval1, err := clientHypergraphStore.LoadHypergraph()
|
||||
assert.NoError(t, err)
|
||||
log.Printf("Generated data")
|
||||
leaves0 := crypto.CompareLeaves(
|
||||
crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
eval0.GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
)
|
||||
leaves1 := crypto.CompareLeaves(
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
eval1.GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
)
|
||||
assert.Len(t, leaves0, 0)
|
||||
assert.Len(t, leaves1, 0)
|
||||
|
||||
lis, err := net.Listen("tcp", ":50051")
|
||||
if err != nil {
|
||||
@ -212,14 +257,22 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
log.Fatalf("Client: failed to stream: %v", err)
|
||||
}
|
||||
|
||||
err = rpc.SyncTreeBidirectionally(str, logger, append(append([]byte{}, shardKey.L1[:]...), shardKey.L2[:]...), protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS, clientHypergraphStore, crdts[1].GetVertexAdds()[shardKey].GetTree(), false)
|
||||
err = rpc.SyncTreeBidirectionally(
|
||||
str,
|
||||
logger,
|
||||
append(append([]byte{}, shardKey.L1[:]...), shardKey.L2[:]...),
|
||||
protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS,
|
||||
clientHypergraphStore,
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree(),
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("Client: failed to sync 1: %v", err)
|
||||
}
|
||||
|
||||
leaves := crypto.CompareLeaves(
|
||||
crdts[0].GetVertexAdds()[shardKey].GetTree(),
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree(),
|
||||
crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
)
|
||||
fmt.Println(len(leaves))
|
||||
|
||||
@ -241,8 +294,8 @@ func TestHypergraphSyncServer(t *testing.T) {
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree().Commit(false),
|
||||
) {
|
||||
leaves := crypto.CompareLeaves(
|
||||
crdts[0].GetVertexAdds()[shardKey].GetTree(),
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree(),
|
||||
crdts[0].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
crdts[1].GetVertexAdds()[shardKey].GetTree().(*store.PersistentVectorTree).GetInternalTree(),
|
||||
)
|
||||
fmt.Println(len(leaves))
|
||||
log.Fatalf(
|
||||
|
||||
@ -104,11 +104,11 @@ type ClockStore interface {
|
||||
minFrameNumber uint64,
|
||||
maxFrameNumber uint64,
|
||||
) error
|
||||
GetDataStateTree(filter []byte) (*crypto.VectorCommitmentTree, error)
|
||||
GetDataStateTree(filter []byte) (*crypto.RawVectorCommitmentTree, error)
|
||||
SetDataStateTree(
|
||||
txn Transaction,
|
||||
filter []byte,
|
||||
tree *crypto.VectorCommitmentTree,
|
||||
tree *crypto.RawVectorCommitmentTree,
|
||||
) error
|
||||
}
|
||||
|
||||
@ -1696,7 +1696,7 @@ func (p *PebbleClockStore) SetProverTriesForFrame(
|
||||
}
|
||||
|
||||
func (p *PebbleClockStore) GetDataStateTree(filter []byte) (
|
||||
*crypto.VectorCommitmentTree,
|
||||
*crypto.RawVectorCommitmentTree,
|
||||
error,
|
||||
) {
|
||||
data, closer, err := p.db.Get(clockDataStateTreeKey(filter))
|
||||
@ -1708,7 +1708,7 @@ func (p *PebbleClockStore) GetDataStateTree(filter []byte) (
|
||||
return nil, errors.Wrap(err, "get data state tree")
|
||||
}
|
||||
defer closer.Close()
|
||||
tree := &crypto.VectorCommitmentTree{}
|
||||
tree := &crypto.RawVectorCommitmentTree{}
|
||||
var b bytes.Buffer
|
||||
b.Write(data)
|
||||
dec := gob.NewDecoder(&b)
|
||||
@ -1722,7 +1722,7 @@ func (p *PebbleClockStore) GetDataStateTree(filter []byte) (
|
||||
func (p *PebbleClockStore) SetDataStateTree(
|
||||
txn Transaction,
|
||||
filter []byte,
|
||||
tree *crypto.VectorCommitmentTree,
|
||||
tree *crypto.RawVectorCommitmentTree,
|
||||
) error {
|
||||
b := new(bytes.Buffer)
|
||||
enc := gob.NewEncoder(b)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user