From 88b914503dc995c5a22e3ad294bc6f6026af5a73 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Wed, 19 Mar 2025 02:04:55 -0500 Subject: [PATCH] lazy hypergraph storage/retrieval --- node/crypto/lazy_proof_tree.go | 1081 +++++++++++++++++ node/crypto/proof_tree.go | 273 ----- node/crypto/tree_compare.go | 163 ++- .../token/token_execution_engine.go | 28 +- .../intrinsics/token/token_genesis.go | 10 - node/hypergraph/application/hypergraph.go | 189 +-- node/rpc/hypergraph_sync_rpc_server.go | 124 +- node/rpc/hypergraph_sync_rpc_server_test.go | 88 +- node/store/hypergraph.go | 776 +++++++++++- 9 files changed, 2239 insertions(+), 493 deletions(-) create mode 100644 node/crypto/lazy_proof_tree.go diff --git a/node/crypto/lazy_proof_tree.go b/node/crypto/lazy_proof_tree.go new file mode 100644 index 0000000..42d3431 --- /dev/null +++ b/node/crypto/lazy_proof_tree.go @@ -0,0 +1,1081 @@ +package crypto + +import ( + "bytes" + "crypto/sha512" + "encoding/binary" + "fmt" + "io" + "math/big" + "slices" + "strings" + "sync" + + "github.com/pkg/errors" + "golang.org/x/crypto/sha3" + rbls48581 "source.quilibrium.com/quilibrium/monorepo/bls48581" + "source.quilibrium.com/quilibrium/monorepo/node/internal/runtime" +) + +type ShardKey struct { + L1 [3]byte + L2 [32]byte +} + +type LazyVectorCommitmentNode interface { + Commit( + setType string, + phaseType string, + shardKey ShardKey, + path []int, + recalculate bool, + ) []byte + GetSize() *big.Int +} + +type LazyVectorCommitmentLeafNode struct { + Key []byte + Value []byte + HashTarget []byte + Commitment []byte + Size *big.Int + Store TreeBackingStore +} + +type LazyVectorCommitmentBranchNode struct { + Prefix []int + Children [BranchNodes]LazyVectorCommitmentNode + Commitment []byte + Size *big.Int + LeafCount int + LongestBranch int + FullPrefix []int + Store TreeBackingStore + FullyLoaded bool +} + +func (n *LazyVectorCommitmentLeafNode) Commit( + setType string, + phaseType string, + shardKey ShardKey, + path []int, + recalculate bool, +) []byte { + if n.Commitment == nil || recalculate { + h := sha512.New() + h.Write([]byte{0}) + h.Write(n.Key) + if len(n.HashTarget) != 0 { + h.Write(n.HashTarget) + } else { + h.Write(n.Value) + } + n.Commitment = h.Sum(nil) + if err := n.Store.InsertNode( + setType, + phaseType, + shardKey, + n.Key, + path, + n, + ); err != nil { + panic(err) + } + } + return n.Commitment +} + +func (n *LazyVectorCommitmentLeafNode) GetSize() *big.Int { + return n.Size +} + +func (n *LazyVectorCommitmentBranchNode) Commit( + setType string, + phaseType string, + shardKey ShardKey, + path []int, + recalculate bool, +) []byte { + if n.Commitment == nil || recalculate { + vector := make([][]byte, len(n.Children)) + wg := sync.WaitGroup{} + throttle := make(chan struct{}, runtime.WorkerCount(0, false)) + for i, child := range n.Children { + throttle <- struct{}{} + wg.Add(1) + go func(i int, child LazyVectorCommitmentNode) { + defer func() { <-throttle }() + defer wg.Done() + + if child == nil { + var err error + child, err = n.Store.GetNodeByPath( + setType, + phaseType, + shardKey, + slices.Concat(n.FullPrefix, []int{i}), + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + } + if child != nil { + out := child.Commit( + setType, + phaseType, + shardKey, + slices.Concat(n.FullPrefix, []int{i}), + recalculate, + ) + switch c := child.(type) { + case *LazyVectorCommitmentBranchNode: + h := sha512.New() + h.Write([]byte{1}) + for _, p := range c.Prefix { + h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p))) + } + h.Write(out) + out = h.Sum(nil) + case *LazyVectorCommitmentLeafNode: + // do nothing + } + vector[i] = out + } else { + vector[i] = make([]byte, 64) + } + }(i, child) + } + wg.Wait() + data := []byte{} + for _, vec := range vector { + data = append(data, vec...) + } + n.Commitment = rbls48581.CommitRaw(data, 64) + if err := n.Store.InsertNode( + setType, + phaseType, + shardKey, + generateKeyFromPath(n.FullPrefix), + n.FullPrefix, + n, + ); err != nil { + panic(err) + } + } + + return n.Commitment +} + +func (n *LazyVectorCommitmentBranchNode) Verify(index int, proof []byte) bool { + data := []byte{} + if n.Commitment == nil { + panic("verify cannot be run on nil commitments") + } else { + child := n.Children[index] + if child != nil { + var out []byte + switch c := child.(type) { + case *LazyVectorCommitmentBranchNode: + out = c.Commitment + h := sha512.New() + h.Write([]byte{1}) + for _, p := range c.Prefix { + h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p))) + } + h.Write(out) + out = h.Sum(nil) + case *LazyVectorCommitmentLeafNode: + out = c.Commitment + } + data = append(data, out...) + } else { + data = append(data, make([]byte, 64)...) + } + } + + return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 64) +} + +func (n *LazyVectorCommitmentBranchNode) GetSize() *big.Int { + return n.Size +} + +func (n *LazyVectorCommitmentBranchNode) Prove(index int) []byte { + data := []byte{} + for _, child := range n.Children { + if child != nil { + var out []byte + switch c := child.(type) { + case *LazyVectorCommitmentBranchNode: + out = c.Commitment + h := sha512.New() + h.Write([]byte{1}) + for _, p := range c.Prefix { + h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p))) + } + h.Write(out) + out = h.Sum(nil) + case *LazyVectorCommitmentLeafNode: + out = c.Commitment + } + data = append(data, out...) + } else { + data = append(data, make([]byte, 64)...) + } + } + + return rbls48581.ProveRaw(data, uint64(index), 64) +} + +type TreeBackingStore interface { + GetNodeByKey( + setType string, + phaseType string, + shardKey ShardKey, + key []byte, + ) (LazyVectorCommitmentNode, error) + GetNodeByPath( + setType string, + phaseType string, + shardKey ShardKey, + path []int, + ) (LazyVectorCommitmentNode, error) + InsertNode( + setType string, + phaseType string, + shardKey ShardKey, + key []byte, + path []int, + node LazyVectorCommitmentNode, + ) error + SaveRoot( + setType string, + phaseType string, + shardKey ShardKey, + node LazyVectorCommitmentNode, + ) error + DeletePath( + setType string, + phaseType string, + shardKey ShardKey, + path []int, + ) error +} + +type LazyVectorCommitmentTree struct { + Root LazyVectorCommitmentNode + SetType string + PhaseType string + ShardKey ShardKey + Store TreeBackingStore +} + +// Insert adds or updates a key-value pair in the tree +func (t *LazyVectorCommitmentTree) Insert( + key, value, hashTarget []byte, + size *big.Int, +) error { + if len(key) == 0 { + return errors.New("empty key not allowed") + } + + var insert func( + node LazyVectorCommitmentNode, + depth int, + path []int, + ) (int, LazyVectorCommitmentNode) + insert = func( + node LazyVectorCommitmentNode, + depth int, + path []int, + ) (int, LazyVectorCommitmentNode) { + if node == nil { + var err error + node, err = t.Store.GetNodeByPath( + t.SetType, + t.PhaseType, + t.ShardKey, + path, + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + } + if node == nil { + newNode := &LazyVectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + Store: t.Store, + } + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + key, + path, + newNode, + ) + if err != nil { + // todo: no panic + panic(err) + } + return 1, newNode + } else { + branch, ok := node.(*LazyVectorCommitmentBranchNode) + if ok && !branch.FullyLoaded { + for i := 0; i < BranchNodes; i++ { + var err error + branch.Children[i], err = t.Store.GetNodeByPath( + t.SetType, + t.PhaseType, + t.ShardKey, + slices.Concat(path, []int{i}), + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + } + branch.FullyLoaded = true + } + } + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + if bytes.Equal(n.Key, key) { + n.Value = value + n.HashTarget = hashTarget + n.Commitment = nil + n.Size = size + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + key, + path, + n, + ) + if err != nil { + // todo: no panic + panic(err) + } + return 0, n + } + + // Get common prefix nibbles and divergence point + sharedNibbles, divergeDepth := getNibblesUntilDiverge(n.Key, key, depth) + + // Create single branch node with shared prefix + branch := &LazyVectorCommitmentBranchNode{ + Prefix: sharedNibbles, + LeafCount: 2, + LongestBranch: 1, + Size: new(big.Int).Add(n.Size, size), + FullPrefix: slices.Concat(path, sharedNibbles), + Store: t.Store, + FullyLoaded: true, + } + + // Add both leaves at their final positions + finalOldNibble := getNextNibble(n.Key, divergeDepth) + finalNewNibble := getNextNibble(key, divergeDepth) + branch.Children[finalOldNibble] = n + branch.Children[finalNewNibble] = &LazyVectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + Store: t.Store, + } + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + n.Key, + slices.Concat(path, sharedNibbles, []int{finalOldNibble}), + n, + ) + if err != nil { + // todo: no panic + panic(err) + } + + err = t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + key, + slices.Concat(path, sharedNibbles, []int{finalNewNibble}), + branch.Children[finalNewNibble], + ) + if err != nil { + // todo: no panic + panic(err) + } + + err = t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + generateKeyFromPath(slices.Concat(path, sharedNibbles)), + slices.Concat(path, sharedNibbles), + branch, + ) + if err != nil { + // todo: no panic + panic(err) + } + + return 1, branch + + case *LazyVectorCommitmentBranchNode: + if len(n.Prefix) > 0 { + // Check if the new key matches the prefix + for i, expectedNibble := range n.Prefix { + actualNibble := getNextNibble(key, depth+i*BranchBits) + if actualNibble != expectedNibble { + // Create new branch with shared prefix subset + newBranch := &LazyVectorCommitmentBranchNode{ + Prefix: n.Prefix[:i], + LeafCount: n.LeafCount + 1, + LongestBranch: n.LongestBranch + 1, + Size: new(big.Int).Add(n.Size, size), + Store: t.Store, + FullPrefix: slices.Concat(path, n.Prefix[:i]), + FullyLoaded: true, + } + // Position old branch and new leaf + newBranch.Children[expectedNibble] = n + n.Prefix = n.Prefix[i+1:] // remove shared prefix from old branch + newBranch.Children[actualNibble] = &LazyVectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + Store: t.Store, + } + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + key, + slices.Concat(path, newBranch.Prefix, []int{actualNibble}), + newBranch.Children[actualNibble], + ) + if err != nil { + // todo: no panic + panic(err) + } + + n.FullPrefix = slices.Concat( + path, + newBranch.Prefix, + []int{expectedNibble}, + ) + err = t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + generateKeyFromPath(slices.Concat(path, newBranch.Prefix, []int{expectedNibble})), + slices.Concat(path, newBranch.Prefix, []int{expectedNibble}), + newBranch.Children[expectedNibble], + ) + if err != nil { + // todo: no panic + panic(err) + } + + err = t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + generateKeyFromPath(slices.Concat(path, newBranch.Prefix)), + slices.Concat(path, newBranch.Prefix), + newBranch, + ) + if err != nil { + // todo: no panic + panic(err) + } + + return 1, newBranch + } + } + + // Key matches prefix, continue with final nibble + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) + maybeBranch, ok := n.Children[finalNibble].(*LazyVectorCommitmentBranchNode) + newPath := slices.Concat(path, n.Prefix, []int{finalNibble}) + if ok { + newPath = append(newPath, maybeBranch.Prefix...) + } + delta, inserted := insert( + n.Children[finalNibble], + depth+len(n.Prefix)*BranchBits+BranchBits, + newPath, + ) + n.Children[finalNibble] = inserted + n.Commitment = nil + n.LeafCount += delta + switch i := inserted.(type) { + case *LazyVectorCommitmentBranchNode: + if n.LongestBranch <= i.LongestBranch { + n.LongestBranch = i.LongestBranch + 1 + } + case *LazyVectorCommitmentLeafNode: + n.LongestBranch = 1 + } + if delta != 0 { + n.Size = n.Size.Add(n.Size, size) + } + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + generateKeyFromPath(path), + path, + n, + ) + if err != nil { + // todo: no panic + panic(err) + } + + return delta, n + } else { + // Simple branch without prefix + nibble := getNextNibble(key, depth) + maybeBranch, ok := n.Children[nibble].(*LazyVectorCommitmentBranchNode) + newPath := slices.Concat(path, n.Prefix, []int{nibble}) + if ok { + newPath = append(newPath, maybeBranch.Prefix...) + } + delta, inserted := insert(n.Children[nibble], depth+BranchBits, newPath) + n.Children[nibble] = inserted + n.Commitment = nil + n.LeafCount += delta + switch i := inserted.(type) { + case *LazyVectorCommitmentBranchNode: + if n.LongestBranch <= i.LongestBranch { + n.LongestBranch = i.LongestBranch + 1 + } + case *LazyVectorCommitmentLeafNode: + n.LongestBranch = 1 + } + if delta != 0 { + n.Size = n.Size.Add(n.Size, size) + } + + err := t.Store.InsertNode( + t.SetType, + t.PhaseType, + t.ShardKey, + generateKeyFromPath(path), + path, + n, + ) + if err != nil { + // todo: no panic + panic(err) + } + + return delta, n + } + } + + return 0, nil + } + + prefix := []int{} + branch, ok := t.Root.(*LazyVectorCommitmentBranchNode) + if ok { + prefix = append(prefix, branch.Prefix...) + } + + _, t.Root = insert(t.Root, 0, prefix) + return errors.Wrap(t.Store.SaveRoot( + t.SetType, + t.PhaseType, + t.ShardKey, + t.Root, + ), "insert") +} + +func generateKeyFromPath(path []int) []byte { + b := []byte{} + for _, p := range path { + b = append(b, byte(p)) + } + hash := sha3.Sum256(b) + return hash[:] +} + +func (t *LazyVectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool { + if len(key) == 0 { + return false + } + + var verify func(node LazyVectorCommitmentNode, proofs [][]byte, depth int) bool + verify = func(node LazyVectorCommitmentNode, proofs [][]byte, depth int) bool { + if node == nil { + return false + } + + if len(proofs) == 0 { + return false + } + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + if bytes.Equal(n.Key, key) { + return bytes.Equal(n.Value, proofs[0]) + } + return false + + case *LazyVectorCommitmentBranchNode: + // Check prefix match + for i, expectedNibble := range n.Prefix { + if getNextNibble(key, depth+i*BranchBits) != expectedNibble { + return false + } + } + + // Get final nibble after prefix + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) + + if !n.Verify(finalNibble, proofs[0]) { + return false + } + + return verify( + n.Children[finalNibble], + proofs[1:], + depth+len(n.Prefix)*BranchBits+BranchBits, + ) + } + + return false + } + + return verify(t.Root, proofs, 0) +} + +func (t *LazyVectorCommitmentTree) Prove(key []byte) [][]byte { + if len(key) == 0 { + return nil + } + + var prove func(node LazyVectorCommitmentNode, depth int) [][]byte + prove = func(node LazyVectorCommitmentNode, depth int) [][]byte { + if node == nil { + return nil + } + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + if bytes.Equal(n.Key, key) { + return [][]byte{n.Value} + } + return nil + + case *LazyVectorCommitmentBranchNode: + // Check prefix match + for i, expectedNibble := range n.Prefix { + if getNextNibble(key, depth+i*BranchBits) != expectedNibble { + return nil + } + } + + // Get final nibble after prefix + finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) + + proofs := [][]byte{n.Prove(finalNibble)} + + return append( + proofs, + prove( + n.Children[finalNibble], + depth+len(n.Prefix)*BranchBits+BranchBits, + )..., + ) + } + + return nil + } + + return prove(t.Root, 0) +} + +// Get retrieves a value from the tree by key +func (t *LazyVectorCommitmentTree) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errors.Wrap(errors.New("empty key not allowed"), "get") + } + + node, err := t.Store.GetNodeByKey(t.SetType, t.PhaseType, t.ShardKey, key) + if err != nil { + return nil, errors.Wrap(err, "get") + } + + leaf, ok := node.(*LazyVectorCommitmentLeafNode) + if !ok { + return nil, errors.Wrap(errors.New("invalid node"), "get") + } + + return leaf.Value, nil +} + +func (t *LazyVectorCommitmentTree) GetMetadata() ( + leafCount int, + longestBranch int, +) { + switch root := t.Root.(type) { + case nil: + return 0, 0 + case *LazyVectorCommitmentLeafNode: + return 1, 0 + case *LazyVectorCommitmentBranchNode: + return root.LeafCount, root.LongestBranch + } + return 0, 0 +} + +// Commit returns the root of the tree +func (t *LazyVectorCommitmentTree) Commit(recalculate bool) []byte { + if t.Root == nil { + return make([]byte, 64) + } + + commitment := t.Root.Commit( + t.SetType, + t.PhaseType, + t.ShardKey, + []int{}, + recalculate, + ) + + err := t.Store.SaveRoot(t.SetType, t.PhaseType, t.ShardKey, t.Root) + if err != nil { + panic(err) + } + + return commitment +} + +func (t *LazyVectorCommitmentTree) GetSize() *big.Int { + return t.Root.GetSize() +} + +func SerializeTree(tree *LazyVectorCommitmentTree) ([]byte, error) { + var buf bytes.Buffer + if err := serializeNode(&buf, tree.Root); err != nil { + return nil, fmt.Errorf("failed to serialize tree: %w", err) + } + return buf.Bytes(), nil +} + +func DeserializeTree( + atomType string, + phaseType string, + shardKey ShardKey, + store TreeBackingStore, + data []byte, +) (*LazyVectorCommitmentTree, error) { + buf := bytes.NewReader(data) + node, err := deserializeNode(store, buf) + if err != nil { + return nil, fmt.Errorf("failed to deserialize tree: %w", err) + } + return &LazyVectorCommitmentTree{ + Root: node, + SetType: atomType, + PhaseType: phaseType, + ShardKey: shardKey, + Store: store, + }, nil +} + +func serializeNode(w io.Writer, node LazyVectorCommitmentNode) error { + if node == nil { + if err := binary.Write(w, binary.BigEndian, TypeNil); err != nil { + return err + } + return nil + } + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + if err := binary.Write(w, binary.BigEndian, TypeLeaf); err != nil { + return err + } + return SerializeLeafNode(w, n) + case *LazyVectorCommitmentBranchNode: + if err := binary.Write(w, binary.BigEndian, TypeBranch); err != nil { + return err + } + return SerializeBranchNode(w, n, true) + default: + return fmt.Errorf("unknown node type: %T", node) + } +} + +func SerializeLeafNode(w io.Writer, node *LazyVectorCommitmentLeafNode) error { + if err := serializeBytes(w, node.Key); err != nil { + return err + } + + if err := serializeBytes(w, node.Value); err != nil { + return err + } + + if err := serializeBytes(w, node.HashTarget); err != nil { + return err + } + + if err := serializeBytes(w, node.Commitment); err != nil { + return err + } + + return serializeBigInt(w, node.Size) +} + +func SerializeBranchNode( + w io.Writer, + node *LazyVectorCommitmentBranchNode, + descend bool, +) error { + if err := serializeIntSlice(w, node.Prefix); err != nil { + return err + } + + if descend { + for i := 0; i < BranchNodes; i++ { + child := node.Children[i] + if err := serializeNode(w, child); err != nil { + return err + } + } + } + + if err := serializeBytes(w, node.Commitment); err != nil { + return err + } + + if err := serializeBigInt(w, node.Size); err != nil { + return err + } + + if err := binary.Write( + w, + binary.BigEndian, + int64(node.LeafCount), + ); err != nil { + return err + } + + return binary.Write(w, binary.BigEndian, int32(node.LongestBranch)) +} + +func deserializeNode( + store TreeBackingStore, + r io.Reader, +) (LazyVectorCommitmentNode, error) { + var nodeType byte + if err := binary.Read(r, binary.BigEndian, &nodeType); err != nil { + return nil, err + } + + switch nodeType { + case TypeNil: + return nil, nil + case TypeLeaf: + return DeserializeLeafNode(store, r) + case TypeBranch: + return DeserializeBranchNode(store, r, true) + default: + return nil, fmt.Errorf("unknown node type marker: %d", nodeType) + } +} + +func DeserializeLeafNode( + store TreeBackingStore, + r io.Reader, +) (*LazyVectorCommitmentLeafNode, error) { + node := &LazyVectorCommitmentLeafNode{} + + key, err := deserializeBytes(r) + if err != nil { + return nil, err + } + node.Key = key + + value, err := deserializeBytes(r) + if err != nil { + return nil, err + } + node.Value = value + + hashTarget, err := deserializeBytes(r) + if err != nil { + return nil, err + } + node.HashTarget = hashTarget + node.Store = store + + commitment, err := deserializeBytes(r) + if err != nil { + return nil, err + } + node.Commitment = commitment + + size, err := deserializeBigInt(r) + if err != nil { + return nil, err + } + node.Size = size + + return node, nil +} + +func DeserializeBranchNode( + store TreeBackingStore, + r io.Reader, + descend bool, +) (*LazyVectorCommitmentBranchNode, error) { + node := &LazyVectorCommitmentBranchNode{} + + prefix, err := deserializeIntSlice(r) + if err != nil { + return nil, err + } + node.Prefix = prefix + node.Store = store + + node.Children = [BranchNodes]LazyVectorCommitmentNode{} + if descend { + for i := 0; i < BranchNodes; i++ { + child, err := deserializeNode(store, r) + if err != nil { + return nil, err + } + node.Children[i] = child + } + } + + commitment, err := deserializeBytes(r) + if err != nil { + return nil, err + } + node.Commitment = commitment + + size, err := deserializeBigInt(r) + if err != nil { + return nil, err + } + node.Size = size + + var leafCount int64 + if err := binary.Read(r, binary.BigEndian, &leafCount); err != nil { + return nil, err + } + node.LeafCount = int(leafCount) + + var longestBranch int32 + if err := binary.Read(r, binary.BigEndian, &longestBranch); err != nil { + return nil, err + } + node.LongestBranch = int(longestBranch) + + return node, nil +} + +func serializeBytes(w io.Writer, data []byte) error { + length := uint64(len(data)) + if err := binary.Write(w, binary.BigEndian, length); err != nil { + return err + } + + if length > 0 { + if _, err := w.Write(data); err != nil { + return err + } + } + return nil +} + +func deserializeBytes(r io.Reader) ([]byte, error) { + var length uint64 + if err := binary.Read(r, binary.BigEndian, &length); err != nil { + return nil, err + } + + if length > 0 { + data := make([]byte, length) + if _, err := io.ReadFull(r, data); err != nil { + return nil, err + } + return data, nil + } + return []byte{}, nil +} + +func serializeIntSlice(w io.Writer, ints []int) error { + length := uint32(len(ints)) + if err := binary.Write(w, binary.BigEndian, length); err != nil { + return err + } + + for _, v := range ints { + if err := binary.Write(w, binary.BigEndian, int32(v)); err != nil { + return err + } + } + return nil +} + +func deserializeIntSlice(r io.Reader) ([]int, error) { + var length uint32 + if err := binary.Read(r, binary.BigEndian, &length); err != nil { + return nil, err + } + + ints := make([]int, length) + for i := range ints { + var v int32 + if err := binary.Read(r, binary.BigEndian, &v); err != nil { + return nil, err + } + ints[i] = int(v) + } + return ints, nil +} + +func serializeBigInt(w io.Writer, n *big.Int) error { + if n == nil { + return binary.Write(w, binary.BigEndian, uint32(0)) + } + + bytes := n.Bytes() + + return serializeBytes(w, bytes) +} + +func deserializeBigInt(r io.Reader) (*big.Int, error) { + bytes, err := deserializeBytes(r) + if err != nil { + return nil, err + } + + if len(bytes) == 0 { + return new(big.Int), nil + } + + n := new(big.Int).SetBytes(bytes) + return n, nil +} diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index a8d48d2..7e25e8d 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -7,7 +7,6 @@ import ( "encoding/gob" "errors" "fmt" - "io" "math/big" "sync" @@ -632,275 +631,3 @@ func DebugNode(node VectorCommitmentNode, depth int, prefix string) { } } } - -func SerializeTree(tree *VectorCommitmentTree) ([]byte, error) { - var buf bytes.Buffer - if err := serializeNode(&buf, tree.Root); err != nil { - return nil, fmt.Errorf("failed to serialize tree: %w", err) - } - return buf.Bytes(), nil -} - -func DeserializeTree(data []byte) (*VectorCommitmentTree, error) { - buf := bytes.NewReader(data) - node, err := deserializeNode(buf) - if err != nil { - return nil, fmt.Errorf("failed to deserialize tree: %w", err) - } - return &VectorCommitmentTree{Root: node}, nil -} - -func serializeNode(w io.Writer, node VectorCommitmentNode) error { - if node == nil { - if err := binary.Write(w, binary.BigEndian, TypeNil); err != nil { - return err - } - return nil - } - - switch n := node.(type) { - case *VectorCommitmentLeafNode: - if err := binary.Write(w, binary.BigEndian, TypeLeaf); err != nil { - return err - } - return serializeLeafNode(w, n) - case *VectorCommitmentBranchNode: - if err := binary.Write(w, binary.BigEndian, TypeBranch); err != nil { - return err - } - return serializeBranchNode(w, n) - default: - return fmt.Errorf("unknown node type: %T", node) - } -} - -func serializeLeafNode(w io.Writer, node *VectorCommitmentLeafNode) error { - if err := serializeBytes(w, node.Key); err != nil { - return err - } - - if err := serializeBytes(w, node.Value); err != nil { - return err - } - - if err := serializeBytes(w, node.HashTarget); err != nil { - return err - } - - if err := serializeBytes(w, node.Commitment); err != nil { - return err - } - - return serializeBigInt(w, node.Size) -} - -func serializeBranchNode(w io.Writer, node *VectorCommitmentBranchNode) error { - if err := serializeIntSlice(w, node.Prefix); err != nil { - return err - } - - for i := 0; i < BranchNodes; i++ { - child := node.Children[i] - if err := serializeNode(w, child); err != nil { - return err - } - } - - if err := serializeBytes(w, node.Commitment); err != nil { - return err - } - - if err := serializeBigInt(w, node.Size); err != nil { - return err - } - - if err := binary.Write(w, binary.BigEndian, int64(node.LeafCount)); err != nil { - return err - } - - return binary.Write(w, binary.BigEndian, int32(node.LongestBranch)) -} - -func deserializeNode(r io.Reader) (VectorCommitmentNode, error) { - var nodeType byte - if err := binary.Read(r, binary.BigEndian, &nodeType); err != nil { - return nil, err - } - - switch nodeType { - case TypeNil: - return nil, nil - case TypeLeaf: - return deserializeLeafNode(r) - case TypeBranch: - return deserializeBranchNode(r) - default: - return nil, fmt.Errorf("unknown node type marker: %d", nodeType) - } -} - -func deserializeLeafNode(r io.Reader) (*VectorCommitmentLeafNode, error) { - node := &VectorCommitmentLeafNode{} - - key, err := deserializeBytes(r) - if err != nil { - return nil, err - } - node.Key = key - - value, err := deserializeBytes(r) - if err != nil { - return nil, err - } - node.Value = value - - hashTarget, err := deserializeBytes(r) - if err != nil { - return nil, err - } - node.HashTarget = hashTarget - - commitment, err := deserializeBytes(r) - if err != nil { - return nil, err - } - node.Commitment = commitment - - size, err := deserializeBigInt(r) - if err != nil { - return nil, err - } - node.Size = size - - return node, nil -} - -func deserializeBranchNode(r io.Reader) (*VectorCommitmentBranchNode, error) { - node := &VectorCommitmentBranchNode{} - - prefix, err := deserializeIntSlice(r) - if err != nil { - return nil, err - } - node.Prefix = prefix - - node.Children = [BranchNodes]VectorCommitmentNode{} - for i := 0; i < BranchNodes; i++ { - child, err := deserializeNode(r) - if err != nil { - return nil, err - } - node.Children[i] = child - } - - commitment, err := deserializeBytes(r) - if err != nil { - return nil, err - } - node.Commitment = commitment - - size, err := deserializeBigInt(r) - if err != nil { - return nil, err - } - node.Size = size - - var leafCount int64 - if err := binary.Read(r, binary.BigEndian, &leafCount); err != nil { - return nil, err - } - node.LeafCount = int(leafCount) - - var longestBranch int32 - if err := binary.Read(r, binary.BigEndian, &longestBranch); err != nil { - return nil, err - } - node.LongestBranch = int(longestBranch) - - return node, nil -} - -func serializeBytes(w io.Writer, data []byte) error { - length := uint64(len(data)) - if err := binary.Write(w, binary.BigEndian, length); err != nil { - return err - } - - if length > 0 { - if _, err := w.Write(data); err != nil { - return err - } - } - return nil -} - -func deserializeBytes(r io.Reader) ([]byte, error) { - var length uint64 - if err := binary.Read(r, binary.BigEndian, &length); err != nil { - return nil, err - } - - if length > 0 { - data := make([]byte, length) - if _, err := io.ReadFull(r, data); err != nil { - return nil, err - } - return data, nil - } - return []byte{}, nil -} - -func serializeIntSlice(w io.Writer, ints []int) error { - length := uint32(len(ints)) - if err := binary.Write(w, binary.BigEndian, length); err != nil { - return err - } - - for _, v := range ints { - if err := binary.Write(w, binary.BigEndian, int32(v)); err != nil { - return err - } - } - return nil -} - -func deserializeIntSlice(r io.Reader) ([]int, error) { - var length uint32 - if err := binary.Read(r, binary.BigEndian, &length); err != nil { - return nil, err - } - - ints := make([]int, length) - for i := range ints { - var v int32 - if err := binary.Read(r, binary.BigEndian, &v); err != nil { - return nil, err - } - ints[i] = int(v) - } - return ints, nil -} - -func serializeBigInt(w io.Writer, n *big.Int) error { - if n == nil { - return binary.Write(w, binary.BigEndian, uint32(0)) - } - - bytes := n.Bytes() - - return serializeBytes(w, bytes) -} - -func deserializeBigInt(r io.Reader) (*big.Int, error) { - bytes, err := deserializeBytes(r) - if err != nil { - return nil, err - } - - if len(bytes) == 0 { - return new(big.Int), nil - } - - n := new(big.Int).SetBytes(bytes) - return n, nil -} diff --git a/node/crypto/tree_compare.go b/node/crypto/tree_compare.go index a52c744..5798202 100644 --- a/node/crypto/tree_compare.go +++ b/node/crypto/tree_compare.go @@ -3,10 +3,12 @@ package crypto import ( "bytes" "fmt" + "slices" + "strings" ) // CompareTreesAtHeight compares two vector commitment trees at each level -func CompareTreesAtHeight(tree1, tree2 *VectorCommitmentTree) [][]ComparisonResult { +func CompareTreesAtHeight(tree1, tree2 *LazyVectorCommitmentTree) [][]ComparisonResult { if tree1 == nil || tree2 == nil { return nil } @@ -31,7 +33,7 @@ type ComparisonResult struct { Matches bool // Whether the commitments match } -func getMaxHeight(node1, node2 VectorCommitmentNode) int { +func getMaxHeight(node1, node2 LazyVectorCommitmentNode) int { height1 := getHeight(node1) height2 := getHeight(node2) if height1 > height2 { @@ -40,15 +42,15 @@ func getMaxHeight(node1, node2 VectorCommitmentNode) int { return height2 } -func getHeight(node VectorCommitmentNode) int { +func getHeight(node LazyVectorCommitmentNode) int { if node == nil { return 0 } switch n := node.(type) { - case *VectorCommitmentLeafNode: + case *LazyVectorCommitmentLeafNode: return 0 - case *VectorCommitmentBranchNode: + case *LazyVectorCommitmentBranchNode: maxChildHeight := 0 for _, child := range n.Children { childHeight := getHeight(child) @@ -61,7 +63,10 @@ func getHeight(node VectorCommitmentNode) int { return 0 } -func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, currentHeight int) []ComparisonResult { +func compareLevelCommits( + node1, node2 LazyVectorCommitmentNode, + targetHeight, currentHeight int, +) []ComparisonResult { if node1 == nil && node2 == nil { return nil } @@ -70,10 +75,26 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren if currentHeight == targetHeight { var commit1, commit2 []byte if node1 != nil { - commit1 = node1.Commit(false) + leaf1, lok := node1.(*LazyVectorCommitmentLeafNode) + branch1, bok := node1.(*LazyVectorCommitmentBranchNode) + if lok { + commit1 = leaf1.Commitment + } else if bok { + commit1 = branch1.Commitment + } else { + panic("invalid node type") + } } if node2 != nil { - commit2 = node2.Commit(false) + leaf2, lok := node1.(*LazyVectorCommitmentLeafNode) + branch2, bok := node1.(*LazyVectorCommitmentBranchNode) + if lok { + commit2 = leaf2.Commitment + } else if bok { + commit2 = branch2.Commitment + } else { + panic("invalid node type") + } } return []ComparisonResult{{ @@ -89,8 +110,8 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren // Handle branch nodes switch n1 := node1.(type) { - case *VectorCommitmentBranchNode: - n2, ok := node2.(*VectorCommitmentBranchNode) + case *LazyVectorCommitmentBranchNode: + n2, ok := node2.(*LazyVectorCommitmentBranchNode) if !ok { // Trees have different structure at this point return results @@ -115,7 +136,7 @@ func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, curren } // TraverseAndCompare provides a channel-based iterator for comparing trees -func TraverseAndCompare(tree1, tree2 *VectorCommitmentTree) chan ComparisonResult { +func TraverseAndCompare(tree1, tree2 *LazyVectorCommitmentTree) chan ComparisonResult { resultChan := make(chan ComparisonResult) go func() { @@ -150,16 +171,28 @@ type LeafDifference struct { } // CompareLeaves returns all leaves that differ between the two trees -func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { +func CompareLeaves(tree1, tree2 *LazyVectorCommitmentTree) []LeafDifference { // Get all leaves from both trees - leaves1 := GetAllLeaves(tree1.Root) - leaves2 := GetAllLeaves(tree2.Root) + leaves1 := GetAllLeaves( + tree1.SetType, + tree1.PhaseType, + tree1.ShardKey, + tree1.Root, + ) + leaves2 := GetAllLeaves( + tree2.SetType, + tree2.PhaseType, + tree2.ShardKey, + tree2.Root, + ) + fmt.Println(len(leaves1)) + fmt.Println(len(leaves2)) differences := make([]LeafDifference, 0) // Use maps for efficient lookup - leafMap1 := make(map[string]*VectorCommitmentLeafNode) - leafMap2 := make(map[string]*VectorCommitmentLeafNode) + leafMap1 := make(map[string]*LazyVectorCommitmentLeafNode) + leafMap2 := make(map[string]*LazyVectorCommitmentLeafNode) // Build maps for _, leaf := range leaves1 { @@ -206,8 +239,10 @@ func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { return differences } -// GetAllLeaves returns all leaf nodes in the tree -func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { +// GetAllPreloadedLeaves returns all leaf nodes in the tree +func GetAllPreloadedLeaves( + node VectorCommitmentNode, +) []*VectorCommitmentLeafNode { if node == nil { return nil } @@ -220,7 +255,93 @@ func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { case *VectorCommitmentBranchNode: for _, child := range n.Children { if child != nil { - childLeaves := GetAllLeaves(child) + childLeaves := GetAllPreloadedLeaves( + child, + ) + leaves = append(leaves, childLeaves...) + } + } + } + + return leaves +} + +func ConvertAllPreloadedLeaves( + atomType string, + phaseType string, + shardKey ShardKey, + store TreeBackingStore, + node LazyVectorCommitmentNode, + path []int, +) []*LazyVectorCommitmentLeafNode { + if node == nil { + return nil + } + + var leaves []*LazyVectorCommitmentLeafNode + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + leaves = append(leaves, n) + case *LazyVectorCommitmentBranchNode: + n.FullPrefix = slices.Concat(path, n.Prefix) + n.FullyLoaded = true + n.Store = store + for i, child := range n.Children { + if child != nil { + childLeaves := ConvertAllPreloadedLeaves( + atomType, + phaseType, + shardKey, + store, + child, + slices.Concat(path, n.Prefix, []int{i}), + ) + leaves = append(leaves, childLeaves...) + } + } + } + + return leaves +} + +// GetAllLeaves returns all leaf nodes in the tree +func GetAllLeaves( + setType string, + phaseType string, + shardKey ShardKey, + node LazyVectorCommitmentNode, +) []*LazyVectorCommitmentLeafNode { + if node == nil { + return nil + } + + var leaves []*LazyVectorCommitmentLeafNode + + switch n := node.(type) { + case *LazyVectorCommitmentLeafNode: + leaves = append(leaves, n) + case *LazyVectorCommitmentBranchNode: + for i, child := range n.Children { + var err error + if child == nil { + child, err = n.Store.GetNodeByPath( + setType, + phaseType, + shardKey, + slices.Concat(n.FullPrefix, []int{i}), + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + } + if child != nil { + childLeaves := GetAllLeaves( + setType, + phaseType, + shardKey, + child, + ) leaves = append(leaves, childLeaves...) } } @@ -231,8 +352,8 @@ func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { func ExampleComparison() { // Create and populate two trees - tree1 := &VectorCommitmentTree{} - tree2 := &VectorCommitmentTree{} + tree1 := &LazyVectorCommitmentTree{} + tree2 := &LazyVectorCommitmentTree{} // Compare trees using channel-based iterator for result := range TraverseAndCompare(tree1, tree2) { diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index b03a994..a1e2a3d 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -161,7 +161,7 @@ func NewTokenExecutionEngine( var inclusionProof *qcrypto.InclusionAggregateProof var proverKeys [][]byte var peerSeniority map[string]uint64 - hg := hypergraph.NewHypergraph() + hg := hypergraph.NewHypergraph(hypergraphStore) mpcithVerEnc := qcrypto.NewMPCitHVerifiableEncryptor( runtime.NumCPU(), ) @@ -910,10 +910,6 @@ func (e *TokenExecutionEngine) hyperSync(totalCoins int) { "hypergraph root commit", zap.String("root", hex.EncodeToString(roots[0])), ) - - if err := e.hypergraphStore.SaveHypergraph(e.hypergraph); err != nil { - e.logger.Error("error while saving", zap.Error(err)) - } } func (e *TokenExecutionEngine) rebuildMissingSetForHypergraph(set [][]byte) { @@ -971,12 +967,6 @@ func (e *TokenExecutionEngine) rebuildMissingSetForHypergraph(set [][]byte) { zap.String("root", fmt.Sprintf("%x", roots[0])), ) - err = e.hypergraphStore.SaveHypergraph(e.hypergraph) - if err != nil { - txn.Abort() - panic(err) - } - if err = txn.Commit(); err != nil { txn.Abort() panic(err) @@ -985,7 +975,7 @@ func (e *TokenExecutionEngine) rebuildMissingSetForHypergraph(set [][]byte) { func (e *TokenExecutionEngine) rebuildHypergraph(totalRange int) { e.logger.Info("rebuilding hypergraph") - e.hypergraph = hypergraph.NewHypergraph() + e.hypergraph = hypergraph.NewHypergraph(e.hypergraphStore) if e.engineConfig.RebuildStart == "" { e.engineConfig.RebuildStart = "0000000000000000000000000000000000000000000000000000000000000000" } @@ -1062,12 +1052,6 @@ func (e *TokenExecutionEngine) rebuildHypergraph(totalRange int) { zap.String("root", fmt.Sprintf("%x", roots[0])), ) - err = e.hypergraphStore.SaveHypergraph(e.hypergraph) - if err != nil { - txn.Abort() - panic(err) - } - if err = txn.Commit(); err != nil { txn.Abort() panic(err) @@ -1755,14 +1739,6 @@ func (e *TokenExecutionEngine) ProcessFrame( zap.String("root", fmt.Sprintf("%x", roots[0])), ) - err = e.hypergraphStore.SaveHypergraph( - hg, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - e.hypergraph = hg return app.Tries, nil diff --git a/node/execution/intrinsics/token/token_genesis.go b/node/execution/intrinsics/token/token_genesis.go index 4c95351..311f28a 100644 --- a/node/execution/intrinsics/token/token_genesis.go +++ b/node/execution/intrinsics/token/token_genesis.go @@ -935,11 +935,6 @@ func CreateGenesisState( } intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - err = hypergraphStore.SaveHypergraph(hg) - if err != nil { - txn.Abort() - panic(err) - } if err = txn.Commit(); err != nil { panic(err) @@ -1100,11 +1095,6 @@ func CreateGenesisState( } } intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - err = hypergraphStore.SaveHypergraph(hg) - if err != nil { - txn.Abort() - panic(err) - } if err := txn.Commit(); err != nil { panic(err) diff --git a/node/hypergraph/application/hypergraph.go b/node/hypergraph/application/hypergraph.go index b319a9e..8d61de2 100644 --- a/node/hypergraph/application/hypergraph.go +++ b/node/hypergraph/application/hypergraph.go @@ -131,7 +131,7 @@ func AtomFromBytes(data []byte) Atom { } extrinsics := make(map[[64]byte]Atom) - for _, a := range crypto.GetAllLeaves(tree) { + for _, a := range crypto.GetAllPreloadedLeaves(tree) { atom := AtomFromBytes(a.Value) extrinsics[[64]byte(a.Key)] = atom } @@ -309,11 +309,6 @@ type ShardAddress struct { L3 [32]byte } -type ShardKey struct { - L1 [3]byte - L2 [32]byte -} - func GetShardAddress(a Atom) ShardAddress { appAddress := a.GetAppAddress() dataAddress := a.GetDataAddress() @@ -325,36 +320,67 @@ func GetShardAddress(a Atom) ShardAddress { } } -func GetShardKey(a Atom) ShardKey { +func GetShardKey(a Atom) crypto.ShardKey { s := GetShardAddress(a) - return ShardKey{L1: s.L1, L2: s.L2} + return crypto.ShardKey{L1: s.L1, L2: s.L2} } type IdSet struct { dirty bool atomType AtomType atoms map[[64]byte]Atom - tree *crypto.VectorCommitmentTree + tree *crypto.LazyVectorCommitmentTree } -func NewIdSet(atomType AtomType) *IdSet { +func NewIdSet( + atomType AtomType, + phaseType PhaseType, + shardKey crypto.ShardKey, + store crypto.TreeBackingStore, +) *IdSet { return &IdSet{ dirty: false, atomType: atomType, atoms: make(map[[64]byte]Atom), - tree: &crypto.VectorCommitmentTree{}, + tree: &crypto.LazyVectorCommitmentTree{ + SetType: string(atomType), + PhaseType: string(phaseType), + ShardKey: shardKey, + Store: store, + }, } } -func (set *IdSet) FromBytes(treeData []byte) error { +func (set *IdSet) FromBytes( + atomType AtomType, + phaseType PhaseType, + shardKey crypto.ShardKey, + store crypto.TreeBackingStore, + treeData []byte, +) ([]Atom, error) { var err error - set.tree, err = crypto.DeserializeTree(treeData) - leaves := crypto.GetAllLeaves(set.tree.Root) + set.tree, err = crypto.DeserializeTree( + string(atomType), + string(phaseType), + shardKey, + store, + treeData, + ) + leaves := crypto.ConvertAllPreloadedLeaves( + string(atomType), + string(phaseType), + shardKey, + store, + set.tree.Root, + []int{}, + ) + atoms := []Atom{} for _, leaf := range leaves { - set.atoms[[64]byte(leaf.Key)] = AtomFromBytes(leaf.Value) + atom := AtomFromBytes(leaf.Value) + atoms = append(atoms, atom) } - return errors.Wrap(err, "from bytes") + return atoms, errors.Wrap(err, "from bytes") } func (set *IdSet) IsDirty() bool { @@ -384,62 +410,48 @@ func (set *IdSet) GetSize() *big.Int { return size } -func (set *IdSet) Delete(atom Atom) bool { - if atom.GetAtomType() != set.atomType { - return false - } - - id := atom.GetID() - if err := set.tree.Delete(id[:]); err != nil { - return false - } - - set.dirty = true - delete(set.atoms, id) - - return true -} - func (set *IdSet) Has(key [64]byte) bool { _, ok := set.atoms[key] return ok } -func (set *IdSet) GetTree() *crypto.VectorCommitmentTree { +func (set *IdSet) GetTree() *crypto.LazyVectorCommitmentTree { return set.tree } type Hypergraph struct { size *big.Int - vertexAdds map[ShardKey]*IdSet - vertexRemoves map[ShardKey]*IdSet - hyperedgeAdds map[ShardKey]*IdSet - hyperedgeRemoves map[ShardKey]*IdSet + vertexAdds map[crypto.ShardKey]*IdSet + vertexRemoves map[crypto.ShardKey]*IdSet + hyperedgeAdds map[crypto.ShardKey]*IdSet + hyperedgeRemoves map[crypto.ShardKey]*IdSet + store crypto.TreeBackingStore } -func NewHypergraph() *Hypergraph { +func NewHypergraph(store crypto.TreeBackingStore) *Hypergraph { return &Hypergraph{ size: big.NewInt(0), - vertexAdds: make(map[ShardKey]*IdSet), - vertexRemoves: make(map[ShardKey]*IdSet), - hyperedgeAdds: make(map[ShardKey]*IdSet), - hyperedgeRemoves: make(map[ShardKey]*IdSet), + vertexAdds: make(map[crypto.ShardKey]*IdSet), + vertexRemoves: make(map[crypto.ShardKey]*IdSet), + hyperedgeAdds: make(map[crypto.ShardKey]*IdSet), + hyperedgeRemoves: make(map[crypto.ShardKey]*IdSet), + store: store, } } -func (hg *Hypergraph) GetVertexAdds() map[ShardKey]*IdSet { +func (hg *Hypergraph) GetVertexAdds() map[crypto.ShardKey]*IdSet { return hg.vertexAdds } -func (hg *Hypergraph) GetVertexRemoves() map[ShardKey]*IdSet { +func (hg *Hypergraph) GetVertexRemoves() map[crypto.ShardKey]*IdSet { return hg.vertexRemoves } -func (hg *Hypergraph) GetHyperedgeAdds() map[ShardKey]*IdSet { +func (hg *Hypergraph) GetHyperedgeAdds() map[crypto.ShardKey]*IdSet { return hg.hyperedgeAdds } -func (hg *Hypergraph) GetHyperedgeRemoves() map[ShardKey]*IdSet { +func (hg *Hypergraph) GetHyperedgeRemoves() map[crypto.ShardKey]*IdSet { return hg.hyperedgeRemoves } @@ -460,15 +472,25 @@ func (hg *Hypergraph) Commit() [][]byte { return commits } -func (hg *Hypergraph) ImportFromBytes( +func (hg *Hypergraph) ImportTree( atomType AtomType, phaseType PhaseType, - shardKey ShardKey, - data []byte, + shardKey crypto.ShardKey, + root crypto.LazyVectorCommitmentNode, + store crypto.TreeBackingStore, ) error { - set := NewIdSet(atomType) - if err := set.FromBytes(data); err != nil { - return errors.Wrap(err, "import from bytes") + set := NewIdSet( + atomType, + phaseType, + shardKey, + store, + ) + set.tree = &crypto.LazyVectorCommitmentTree{ + Root: root, + SetType: string(atomType), + PhaseType: string(phaseType), + ShardKey: shardKey, + Store: store, } switch atomType { @@ -500,16 +522,27 @@ func (hg *Hypergraph) GetSize() *big.Int { } func (hg *Hypergraph) getOrCreateIdSet( - shardAddr ShardKey, - addMap map[ShardKey]*IdSet, - removeMap map[ShardKey]*IdSet, + shardAddr crypto.ShardKey, + addMap map[crypto.ShardKey]*IdSet, + removeMap map[crypto.ShardKey]*IdSet, atomType AtomType, + phaseType PhaseType, ) (*IdSet, *IdSet) { if _, ok := addMap[shardAddr]; !ok { - addMap[shardAddr] = NewIdSet(atomType) + addMap[shardAddr] = NewIdSet( + atomType, + phaseType, + shardAddr, + hg.store, + ) } if _, ok := removeMap[shardAddr]; !ok { - removeMap[shardAddr] = NewIdSet(atomType) + removeMap[shardAddr] = NewIdSet( + atomType, + phaseType, + shardAddr, + hg.store, + ) } return addMap[shardAddr], removeMap[shardAddr] } @@ -521,6 +554,7 @@ func (hg *Hypergraph) AddVertex(v Vertex) error { hg.vertexAdds, hg.vertexRemoves, VertexAtomType, + AddsPhaseType, ) hg.size.Add(hg.size, v.GetSize()) return errors.Wrap(addSet.Add(v), "add vertex") @@ -536,6 +570,7 @@ func (hg *Hypergraph) AddHyperedge(h Hyperedge) error { hg.hyperedgeAdds, hg.hyperedgeRemoves, HyperedgeAtomType, + AddsPhaseType, ) id := h.GetID() if !removeSet.Has(id) { @@ -553,6 +588,7 @@ func (hg *Hypergraph) RemoveVertex(v Vertex) error { hg.vertexAdds, hg.vertexRemoves, VertexAtomType, + AddsPhaseType, ) if err := addSet.Add(v); err != nil { return errors.Wrap(err, "remove vertex") @@ -576,6 +612,7 @@ func (hg *Hypergraph) RemoveVertex(v Vertex) error { hg.vertexAdds, hg.vertexRemoves, VertexAtomType, + RemovesPhaseType, ) hg.size.Sub(hg.size, v.GetSize()) err := removeSet.Add(v) @@ -591,6 +628,7 @@ func (hg *Hypergraph) RemoveHyperedge(h Hyperedge) error { hg.hyperedgeAdds, hg.hyperedgeRemoves, HyperedgeAtomType, + AddsPhaseType, ) if err := addSet.Add(h); err != nil { return errors.Wrap(err, "remove hyperedge") @@ -614,6 +652,7 @@ func (hg *Hypergraph) RemoveHyperedge(h Hyperedge) error { hg.hyperedgeAdds, hg.hyperedgeRemoves, HyperedgeAtomType, + RemovesPhaseType, ) hg.size.Sub(hg.size, h.GetSize()) err := removeSet.Add(h) @@ -627,6 +666,7 @@ func (hg *Hypergraph) LookupVertex(v Vertex) bool { hg.vertexAdds, hg.vertexRemoves, VertexAtomType, + AddsPhaseType, ) id := v.GetID() return addSet.Has(id) && !removeSet.Has(id) @@ -639,6 +679,7 @@ func (hg *Hypergraph) LookupHyperedge(h Hyperedge) bool { hg.hyperedgeAdds, hg.hyperedgeRemoves, HyperedgeAtomType, + AddsPhaseType, ) id := h.GetID() return hg.LookupAtomSet(&h.(*hyperedge).extrinsics) && addSet.Has(id) && !removeSet.Has(id) @@ -680,37 +721,3 @@ func (hg *Hypergraph) Within(a, h Atom) bool { } return false } - -func (hg *Hypergraph) GetReconciledVertexSetForShard( - shardKey ShardKey, -) *IdSet { - vertices := NewIdSet(VertexAtomType) - - if addSet, ok := hg.vertexAdds[shardKey]; ok { - removeSet := hg.vertexRemoves[shardKey] - for id, v := range addSet.atoms { - if !removeSet.Has(id) { - vertices.Add(v) - } - } - } - - return vertices -} - -func (hg *Hypergraph) GetReconciledHyperedgeSetForShard( - shardKey ShardKey, -) *IdSet { - hyperedges := NewIdSet(HyperedgeAtomType) - - if addSet, ok := hg.hyperedgeAdds[shardKey]; ok { - removeSet := hg.hyperedgeRemoves[shardKey] - for _, h := range addSet.atoms { - if !removeSet.Has(h.GetID()) { - hyperedges.Add(h) - } - } - } - - return hyperedges -} diff --git a/node/rpc/hypergraph_sync_rpc_server.go b/node/rpc/hypergraph_sync_rpc_server.go index f079e74..6906e42 100644 --- a/node/rpc/hypergraph_sync_rpc_server.go +++ b/node/rpc/hypergraph_sync_rpc_server.go @@ -9,6 +9,7 @@ import ( "io" "os" "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -238,11 +239,6 @@ func (s *StandaloneHypersyncClient) Start( zap.String("root", hex.EncodeToString(roots[0])), ) - logger.Info("saving hypergraph") - if err := hypergraphStore.SaveHypergraph(hypergraph); err != nil { - logger.Error("error while saving", zap.Error(err)) - } - logger.Info("hypergraph saved") s.done <- syscall.SIGINT } @@ -309,7 +305,7 @@ type streamManager struct { logger *zap.Logger stream HyperStream hypergraphStore store.HypergraphStore - localTree *crypto.VectorCommitmentTree + localTree *crypto.LazyVectorCommitmentTree lastSent time.Time } @@ -319,7 +315,7 @@ func (s *streamManager) sendLeafData( path []int32, metadataOnly bool, ) error { - send := func(leaf *crypto.VectorCommitmentLeafNode) error { + send := func(leaf *crypto.LazyVectorCommitmentLeafNode) error { update := &protobufs.LeafData{ Key: leaf.Key, Value: leaf.Value, @@ -369,10 +365,22 @@ func (s *streamManager) sendLeafData( default: } - node := getNodeAtPath(s.localTree.Root, path, 0) - leaf, ok := node.(*crypto.VectorCommitmentLeafNode) + node := getNodeAtPath( + s.localTree.SetType, + s.localTree.PhaseType, + s.localTree.ShardKey, + s.localTree.Root, + path, + 0, + ) + leaf, ok := node.(*crypto.LazyVectorCommitmentLeafNode) if !ok { - children := crypto.GetAllLeaves(node) + children := crypto.GetAllLeaves( + s.localTree.SetType, + s.localTree.PhaseType, + s.localTree.ShardKey, + node, + ) for _, child := range children { if child == nil { continue @@ -393,10 +401,13 @@ func (s *streamManager) sendLeafData( // the node found (or nil if not found). The depth argument is used for internal // recursion. func getNodeAtPath( - node crypto.VectorCommitmentNode, + setType string, + phaseType string, + shardKey crypto.ShardKey, + node crypto.LazyVectorCommitmentNode, path []int32, depth int, -) crypto.VectorCommitmentNode { +) crypto.LazyVectorCommitmentNode { if node == nil { return nil } @@ -405,9 +416,9 @@ func getNodeAtPath( } switch n := node.(type) { - case *crypto.VectorCommitmentLeafNode: + case *crypto.LazyVectorCommitmentLeafNode: return node - case *crypto.VectorCommitmentBranchNode: + case *crypto.LazyVectorCommitmentBranchNode: // Check that the branch's prefix matches the beginning of the query path. if len(path) < len(n.Prefix) { return nil @@ -431,12 +442,28 @@ func getNodeAtPath( return nil } - child := n.Children[childIndex] + child, err := n.Store.GetNodeByPath( + setType, + phaseType, + shardKey, + slices.Concat(n.FullPrefix, []int{int(childIndex)}), + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + if child == nil { return nil } - return getNodeAtPath(child, remainder[1:], depth+len(n.Prefix)+1) + return getNodeAtPath( + setType, + phaseType, + shardKey, + child, + remainder[1:], + depth+len(n.Prefix)+1, + ) } return nil } @@ -444,29 +471,68 @@ func getNodeAtPath( // getBranchInfoFromTree looks up the node at the given path in the local tree, // computes its commitment, and (if it is a branch) collects its immediate // children's commitments. -func getBranchInfoFromTree(tree *crypto.VectorCommitmentTree, path []int32) ( +func getBranchInfoFromTree( + tree *crypto.LazyVectorCommitmentTree, + path []int32, +) ( *protobufs.HypergraphComparisonResponse, error, ) { - node := getNodeAtPath(tree.Root, path, 0) + node := getNodeAtPath( + tree.SetType, + tree.PhaseType, + tree.ShardKey, + tree.Root, + path, + 0, + ) if node == nil { return nil, fmt.Errorf("node not found at path %v", path) } - commitment := node.Commit(false) + intpath := []int{} + for _, p := range path { + intpath = append(intpath, int(p)) + } + commitment := node.Commit( + tree.SetType, + tree.PhaseType, + tree.ShardKey, + intpath, + false, + ) branchInfo := &protobufs.HypergraphComparisonResponse{ Path: path, Commitment: commitment, IsRoot: len(path) == 0, } - if branch, ok := node.(*crypto.VectorCommitmentBranchNode); ok { + if branch, ok := node.(*crypto.LazyVectorCommitmentBranchNode); ok { for _, p := range branch.Prefix { branchInfo.Path = append(branchInfo.Path, int32(p)) } for i := 0; i < len(branch.Children); i++ { - if branch.Children[i] != nil { - childCommit := branch.Children[i].Commit(false) + child := branch.Children[i] + if child == nil { + var err error + child, err = branch.Store.GetNodeByPath( + tree.SetType, + tree.PhaseType, + tree.ShardKey, + slices.Concat(branch.FullPrefix, []int{i}), + ) + if err != nil && !strings.Contains(err.Error(), "item not found") { + panic(err) + } + } + if child != nil { + childCommit := child.Commit( + tree.SetType, + tree.PhaseType, + tree.ShardKey, + slices.Concat(branch.FullPrefix, []int{i}), + false, + ) branchInfo.Children = append( branchInfo.Children, &protobufs.BranchChild{ @@ -532,7 +598,7 @@ func handleQueryNext( ctx context.Context, incomingQueries <-chan *protobufs.HypergraphComparisonQuery, stream HyperStream, - localTree *crypto.VectorCommitmentTree, + localTree *crypto.LazyVectorCommitmentTree, path []int32, ) ( *protobufs.HypergraphComparisonResponse, @@ -587,7 +653,7 @@ func descendIndex( ctx context.Context, incomingResponses <-chan *protobufs.HypergraphComparisonResponse, stream HyperStream, - localTree *crypto.VectorCommitmentTree, + localTree *crypto.LazyVectorCommitmentTree, path []int32, ) ( *protobufs.HypergraphComparisonResponse, @@ -928,7 +994,7 @@ func syncTreeBidirectionallyServer( logger.Info("received initialization message") // Get the appropriate phase set - var phaseSet map[hypergraph.ShardKey]*hypergraph.IdSet + var phaseSet map[crypto.ShardKey]*hypergraph.IdSet switch query.PhaseSet { case protobufs.HypergraphPhaseSet_HYPERGRAPH_PHASE_SET_VERTEX_ADDS: phaseSet = localHypergraph.GetVertexAdds() @@ -944,7 +1010,7 @@ func syncTreeBidirectionallyServer( return errors.New("invalid shard key") } - shardKey := hypergraph.ShardKey{ + shardKey := crypto.ShardKey{ L1: [3]byte(query.ShardKey[:3]), L2: [32]byte(query.ShardKey[3:]), } @@ -1126,12 +1192,6 @@ outer: zap.String("root", hex.EncodeToString(roots[0])), ) - logger.Info("saving hypergraph") - if err = localHypergraphStore.SaveHypergraph(localHypergraph); err != nil { - logger.Error("error while saving", zap.Error(err)) - } - logger.Info("hypergraph saved") - total, _ := idSet.GetTree().GetMetadata() logger.Info( "current progress, ready to resume connections", diff --git a/node/rpc/hypergraph_sync_rpc_server_test.go b/node/rpc/hypergraph_sync_rpc_server_test.go index a38e698..0099cc0 100644 --- a/node/rpc/hypergraph_sync_rpc_server_test.go +++ b/node/rpc/hypergraph_sync_rpc_server_test.go @@ -10,6 +10,7 @@ import ( "math/big" "net" "testing" + "time" "github.com/cloudflare/circl/sign/ed448" pcrypto "github.com/libp2p/go-libp2p/core/crypto" @@ -42,9 +43,36 @@ type Operation struct { Hyperedge application.Hyperedge } +func TestLoadHypergraphFallback(t *testing.T) { + clientKvdb := store.NewInMemKVDB() + serverKvdb := store.NewInMemKVDB() + logger, _ := zap.NewProduction() + clientHypergraphStore := store.NewPebbleHypergraphStore( + &config.DBConfig{Path: ".configtestclient/store"}, + clientKvdb, + logger, + ) + serverHypergraphStore := store.NewPebbleHypergraphStore( + &config.DBConfig{Path: ".configtestserver/store"}, + serverKvdb, + logger, + ) + + serverLoad, err := serverHypergraphStore.LoadHypergraph() + assert.NoError(t, err) + clientLoad, err := clientHypergraphStore.LoadHypergraph() + assert.NoError(t, err) + for k, a := range serverLoad.GetVertexAdds() { + assert.Equal(t, len(crypto.ConvertAllPreloadedLeaves(string(application.VertexAtomType), string(application.AddsPhaseType), k, serverHypergraphStore, a.GetTree().Root, []int{})), 100000) + } + for k, a := range clientLoad.GetVertexAdds() { + assert.Equal(t, len(crypto.ConvertAllPreloadedLeaves(string(application.VertexAtomType), string(application.AddsPhaseType), k, clientHypergraphStore, a.GetTree().Root, []int{})), 100000) + } +} + func TestHypergraphSyncServer(t *testing.T) { numParties := 3 - numOperations := 10000 + numOperations := 1000 log.Printf("Generating data") enc := crypto.NewMPCitHVerifiableEncryptor(1) pub, _, _ := ed448.GenerateKey(rand.Reader) @@ -103,6 +131,7 @@ func TestHypergraphSyncServer(t *testing.T) { clientKvdb := store.NewInMemKVDB() serverKvdb := store.NewInMemKVDB() + controlKvdb := store.NewInMemKVDB() logger, _ := zap.NewProduction() clientHypergraphStore := store.NewPebbleHypergraphStore( &config.DBConfig{Path: ".configtestclient/store"}, @@ -114,36 +143,46 @@ func TestHypergraphSyncServer(t *testing.T) { serverKvdb, logger, ) + controlHypergraphStore := store.NewPebbleHypergraphStore( + &config.DBConfig{Path: ".configtestcontrol/store"}, + controlKvdb, + logger, + ) crdts := make([]*application.Hypergraph, numParties) - for i := 0; i < numParties; i++ { - crdts[i] = application.NewHypergraph() - } + crdts[0] = application.NewHypergraph(serverHypergraphStore) + crdts[1] = application.NewHypergraph(clientHypergraphStore) + crdts[2] = application.NewHypergraph(controlHypergraphStore) txn, _ := serverHypergraphStore.NewTransaction(false) for _, op := range operations1[:numOperations/2] { switch op.Type { case "AddVertex": id := op.Vertex.GetID() + fmt.Printf("server add vertex %x %v\n", id, time.Now()) serverHypergraphStore.SaveVertexTree(txn, id[:], dataTree) crdts[0].AddVertex(op.Vertex) case "RemoveVertex": crdts[0].RemoveVertex(op.Vertex) case "AddHyperedge": + fmt.Printf("server add hyperedge %v\n", time.Now()) crdts[0].AddHyperedge(op.Hyperedge) case "RemoveHyperedge": + fmt.Printf("server remove hyperedge %v\n", time.Now()) crdts[0].RemoveHyperedge(op.Hyperedge) } } txn.Commit() - for _, op := range operations2[:500] { + for _, op := range operations2[:50] { switch op.Type { case "AddVertex": crdts[0].AddVertex(op.Vertex) case "RemoveVertex": crdts[0].RemoveVertex(op.Vertex) case "AddHyperedge": + fmt.Printf("server add hyperedge %v\n", time.Now()) crdts[0].AddHyperedge(op.Hyperedge) case "RemoveHyperedge": + fmt.Printf("server remove hyperedge %v\n", time.Now()) crdts[0].RemoveHyperedge(op.Hyperedge) } } @@ -153,26 +192,31 @@ func TestHypergraphSyncServer(t *testing.T) { switch op.Type { case "AddVertex": id := op.Vertex.GetID() + fmt.Printf("client add vertex %x %v\n", id, time.Now()) clientHypergraphStore.SaveVertexTree(txn, id[:], dataTree) crdts[1].AddVertex(op.Vertex) case "RemoveVertex": crdts[1].RemoveVertex(op.Vertex) case "AddHyperedge": + fmt.Printf("client add hyperedge %v\n", time.Now()) crdts[1].AddHyperedge(op.Hyperedge) case "RemoveHyperedge": + fmt.Printf("client remove hyperedge %v\n", time.Now()) crdts[1].RemoveHyperedge(op.Hyperedge) } } txn.Commit() - for _, op := range operations2[500:] { + for _, op := range operations2[50:] { switch op.Type { case "AddVertex": crdts[1].AddVertex(op.Vertex) case "RemoveVertex": crdts[1].RemoveVertex(op.Vertex) case "AddHyperedge": + fmt.Printf("client add hyperedge %v\n", time.Now()) crdts[1].AddHyperedge(op.Hyperedge) case "RemoveHyperedge": + fmt.Printf("client remove hyperedge %v\n", time.Now()) crdts[1].RemoveHyperedge(op.Hyperedge) } } @@ -205,22 +249,22 @@ func TestHypergraphSyncServer(t *testing.T) { crdts[0].Commit() crdts[1].Commit() crdts[2].Commit() - err := serverHypergraphStore.SaveHypergraph(crdts[0]) - assert.NoError(t, err) - err = clientHypergraphStore.SaveHypergraph(crdts[1]) - assert.NoError(t, err) - serverLoad, err := serverHypergraphStore.LoadHypergraph() - assert.NoError(t, err) - clientLoad, err := clientHypergraphStore.LoadHypergraph() - assert.NoError(t, err) - assert.Len(t, crypto.CompareLeaves( - crdts[0].GetVertexAdds()[shardKey].GetTree(), - serverLoad.GetVertexAdds()[shardKey].GetTree(), - ), 0) - assert.Len(t, crypto.CompareLeaves( - crdts[1].GetVertexAdds()[shardKey].GetTree(), - clientLoad.GetVertexAdds()[shardKey].GetTree(), - ), 0) + // err := serverHypergraphStore.SaveHypergraph(crdts[0]) + // assert.NoError(t, err) + // err = clientHypergraphStore.SaveHypergraph(crdts[1]) + // assert.NoError(t, err) + // serverLoad, err := serverHypergraphStore.LoadHypergraph() + // assert.NoError(t, err) + // clientLoad, err := clientHypergraphStore.LoadHypergraph() + // assert.NoError(t, err) + // assert.Len(t, crypto.CompareLeaves( + // crdts[0].GetVertexAdds()[shardKey].GetTree(), + // serverLoad.GetVertexAdds()[shardKey].GetTree(), + // ), 0) + // assert.Len(t, crypto.CompareLeaves( + // crdts[1].GetVertexAdds()[shardKey].GetTree(), + // clientLoad.GetVertexAdds()[shardKey].GetTree(), + // ), 0) log.Printf("Generated data") lis, err := net.Listen("tcp", ":50051") diff --git a/node/store/hypergraph.go b/node/store/hypergraph.go index 4a2e615..d153290 100644 --- a/node/store/hypergraph.go +++ b/node/store/hypergraph.go @@ -2,6 +2,7 @@ package store import ( "bytes" + "encoding/binary" "encoding/gob" "encoding/hex" "io/fs" @@ -10,6 +11,7 @@ import ( "path/filepath" "strings" + "github.com/cockroachdb/pebble" "github.com/pkg/errors" "go.uber.org/zap" "source.quilibrium.com/quilibrium/monorepo/node/config" @@ -41,6 +43,38 @@ type HypergraphStore interface { SaveHypergraph( hg *application.Hypergraph, ) error + GetNodeByKey( + setType string, + phaseType string, + shardKey crypto.ShardKey, + key []byte, + ) (crypto.LazyVectorCommitmentNode, error) + GetNodeByPath( + setType string, + phaseType string, + shardKey crypto.ShardKey, + path []int, + ) (crypto.LazyVectorCommitmentNode, error) + InsertNode( + setType string, + phaseType string, + shardKey crypto.ShardKey, + key []byte, + path []int, + node crypto.LazyVectorCommitmentNode, + ) error + SaveRoot( + setType string, + phaseType string, + shardKey crypto.ShardKey, + node crypto.LazyVectorCommitmentNode, + ) error + DeletePath( + setType string, + phaseType string, + shardKey crypto.ShardKey, + path []int, + ) error } var _ HypergraphStore = (*PebbleHypergraphStore)(nil) @@ -64,15 +98,27 @@ func NewPebbleHypergraphStore( } const ( - HYPERGRAPH_SHARD = 0x09 - VERTEX_ADDS = 0x00 - VERTEX_REMOVES = 0x10 - VERTEX_DATA = 0xF0 - HYPEREDGE_ADDS = 0x01 - HYPEREDGE_REMOVES = 0x11 + HYPERGRAPH_SHARD = 0x09 + VERTEX_ADDS = 0x00 + VERTEX_REMOVES = 0x10 + VERTEX_DATA = 0xF0 + HYPEREDGE_ADDS = 0x01 + HYPEREDGE_REMOVES = 0x11 + VERTEX_ADDS_TREE_NODE = 0x02 + VERTEX_REMOVES_TREE_NODE = 0x12 + HYPEREDGE_ADDS_TREE_NODE = 0x03 + HYPEREDGE_REMOVES_TREE_NODE = 0x13 + VERTEX_ADDS_TREE_NODE_BY_PATH = 0x22 + VERTEX_REMOVES_TREE_NODE_BY_PATH = 0x32 + HYPEREDGE_ADDS_TREE_NODE_BY_PATH = 0x23 + HYPEREDGE_REMOVES_TREE_NODE_BY_PATH = 0x33 + VERTEX_ADDS_TREE_ROOT = 0xFC + VERTEX_REMOVES_TREE_ROOT = 0xFD + HYPEREDGE_ADDS_TREE_ROOT = 0xFE + HYPEREDGE_REMOVES_TREE_ROOT = 0xFF ) -func hypergraphVertexAddsKey(shardKey application.ShardKey) []byte { +func hypergraphVertexAddsKey(shardKey crypto.ShardKey) []byte { key := []byte{HYPERGRAPH_SHARD, VERTEX_ADDS} key = append(key, shardKey.L1[:]...) key = append(key, shardKey.L2[:]...) @@ -85,29 +131,161 @@ func hypergraphVertexDataKey(id []byte) []byte { return key } -func hypergraphVertexRemovesKey(shardKey application.ShardKey) []byte { +func hypergraphVertexRemovesKey(shardKey crypto.ShardKey) []byte { key := []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES} key = append(key, shardKey.L1[:]...) key = append(key, shardKey.L2[:]...) return key } -func hypergraphHyperedgeAddsKey(shardKey application.ShardKey) []byte { +func hypergraphHyperedgeAddsKey(shardKey crypto.ShardKey) []byte { key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS} key = append(key, shardKey.L1[:]...) key = append(key, shardKey.L2[:]...) return key } -func hypergraphHyperedgeRemovesKey(shardKey application.ShardKey) []byte { +func hypergraphHyperedgeRemovesKey(shardKey crypto.ShardKey) []byte { key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES} key = append(key, shardKey.L1[:]...) key = append(key, shardKey.L2[:]...) return key } -func shardKeyFromKey(key []byte) application.ShardKey { - return application.ShardKey{ +func hypergraphVertexAddsTreeNodeKey( + shardKey crypto.ShardKey, + nodeKey []byte, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_ADDS_TREE_NODE} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + key = append(key, nodeKey...) + return key +} + +func hypergraphVertexRemovesTreeNodeKey( + shardKey crypto.ShardKey, + nodeKey []byte, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES_TREE_NODE} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + key = append(key, nodeKey...) + return key +} + +func hypergraphHyperedgeAddsTreeNodeKey( + shardKey crypto.ShardKey, + nodeKey []byte, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS_TREE_NODE} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + key = append(key, nodeKey...) + return key +} + +func hypergraphHyperedgeRemovesTreeNodeKey( + shardKey crypto.ShardKey, + nodeKey []byte, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES_TREE_NODE} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + key = append(key, nodeKey...) + return key +} + +func hypergraphVertexAddsTreeNodeByPathKey( + shardKey crypto.ShardKey, + path []int, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_ADDS_TREE_NODE_BY_PATH} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + for _, p := range path { + key = binary.BigEndian.AppendUint64(key, uint64(p)) + } + return key +} + +func hypergraphVertexRemovesTreeNodeByPathKey( + shardKey crypto.ShardKey, + path []int, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES_TREE_NODE_BY_PATH} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + for _, p := range path { + key = binary.BigEndian.AppendUint64(key, uint64(p)) + } + return key +} + +func hypergraphHyperedgeAddsTreeNodeByPathKey( + shardKey crypto.ShardKey, + path []int, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS_TREE_NODE_BY_PATH} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + for _, p := range path { + key = binary.BigEndian.AppendUint64(key, uint64(p)) + } + return key +} + +func hypergraphHyperedgeRemovesTreeNodeByPathKey( + shardKey crypto.ShardKey, + path []int, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES_TREE_NODE_BY_PATH} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + for _, p := range path { + key = binary.BigEndian.AppendUint64(key, uint64(p)) + } + return key +} + +func hypergraphVertexAddsTreeRootKey( + shardKey crypto.ShardKey, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_ADDS_TREE_ROOT} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphVertexRemovesTreeRootKey( + shardKey crypto.ShardKey, +) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES_TREE_ROOT} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphHyperedgeAddsTreeRootKey( + shardKey crypto.ShardKey, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS_TREE_ROOT} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphHyperedgeRemovesTreeRootKey( + shardKey crypto.ShardKey, +) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES_TREE_ROOT} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func shardKeyFromKey(key []byte) crypto.ShardKey { + return crypto.ShardKey{ L1: [3]byte(key[2:5]), L2: [32]byte(key[5:]), } @@ -160,7 +338,7 @@ func (p *PebbleHypergraphStore) LoadVertexData(id []byte) ( } encData := []application.Encrypted{} - for _, d := range crypto.GetAllLeaves(tree) { + for _, d := range crypto.GetAllPreloadedLeaves(tree) { verencData := crypto.MPCitHVerEncFromBytes(d.Value) encData = append(encData, verencData) } @@ -209,9 +387,241 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( *application.Hypergraph, error, ) { - hg := application.NewHypergraph() + hg := application.NewHypergraph(p) hypergraphDir := path.Join(p.config.Path, "hypergraph") + vertexAddsIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, VERTEX_ADDS_TREE_ROOT}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES_TREE_ROOT}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer vertexAddsIter.Close() + + loadedFromDB := false + for vertexAddsIter.First(); vertexAddsIter.Valid(); vertexAddsIter.Next() { + loadedFromDB = true + shardKey := shardKeyFromKey(vertexAddsIter.Key()) + data := vertexAddsIter.Value() + + var node crypto.LazyVectorCommitmentNode + switch data[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode(p, bytes.NewReader(data[1:])) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(data[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(data[5+(pathLength*4):]), + false, + ) + + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(data[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + + err = hg.ImportTree( + application.VertexAtomType, + application.AddsPhaseType, + shardKey, + node, + p, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + vertexRemovesIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES_TREE_ROOT}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS_TREE_ROOT}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer vertexRemovesIter.Close() + + for vertexRemovesIter.First(); vertexRemovesIter.Valid(); vertexRemovesIter.Next() { + loadedFromDB = true + shardKey := shardKeyFromKey(vertexRemovesIter.Key()) + data := vertexRemovesIter.Value() + + var node crypto.LazyVectorCommitmentNode + switch data[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode(p, bytes.NewReader(data[1:])) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(data[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(data[5+(pathLength*4):]), + false, + ) + + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(data[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + + err = hg.ImportTree( + application.VertexAtomType, + application.RemovesPhaseType, + shardKey, + node, + p, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + hyperedgeAddsIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS_TREE_ROOT}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES_TREE_ROOT}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer hyperedgeAddsIter.Close() + + for hyperedgeAddsIter.First(); hyperedgeAddsIter.Valid(); hyperedgeAddsIter.Next() { + loadedFromDB = true + shardKey := shardKeyFromKey(hyperedgeAddsIter.Key()) + data := hyperedgeAddsIter.Value() + + var node crypto.LazyVectorCommitmentNode + switch data[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode( + p, + bytes.NewReader(data[1:]), + ) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(data[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(data[5+(pathLength*4):]), + false, + ) + + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(data[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + + err = hg.ImportTree( + application.HyperedgeAtomType, + application.AddsPhaseType, + shardKey, + node, + p, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + hyperedgeRemovesIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES_TREE_ROOT}, + []byte{(HYPERGRAPH_SHARD + 1), 0x00}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer hyperedgeRemovesIter.Close() + + for hyperedgeRemovesIter.First(); hyperedgeRemovesIter.Valid(); hyperedgeRemovesIter.Next() { + loadedFromDB = true + shardKey := shardKeyFromKey(hyperedgeRemovesIter.Key()) + data := hyperedgeRemovesIter.Value() + + var node crypto.LazyVectorCommitmentNode + switch data[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode(p, bytes.NewReader(data[1:])) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(data[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(data[5+(pathLength*4):]), + false, + ) + + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(data[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + + err = hg.ImportTree( + application.HyperedgeAtomType, + application.RemovesPhaseType, + shardKey, + node, + p, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + if loadedFromDB { + return hg, nil + } + vertexAddsPrefix := hex.EncodeToString( []byte{HYPERGRAPH_SHARD, VERTEX_ADDS}, ) @@ -225,10 +635,11 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES}, ) - return hg, errors.Wrap( + p.logger.Info("converting hypergraph, this may take a moment") + err = errors.Wrap( filepath.WalkDir( hypergraphDir, - func(p string, d fs.DirEntry, err error) error { + func(pa string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -264,26 +675,42 @@ func (p *PebbleHypergraphStore) LoadHypergraph() ( setType = application.RemovesPhaseType } - fileBytes, err := os.ReadFile(p) + fileBytes, err := os.ReadFile(pa) if err != nil { return err } - err = hg.ImportFromBytes( + set := application.NewIdSet( atomType, setType, shardKeyFromKey(shardSet), + p, + ) + atoms, err := set.FromBytes( + atomType, + setType, + shardKeyFromKey(shardSet), + p, fileBytes, ) if err != nil { return err } + for _, atom := range atoms { + hg.AddVertex(atom.(application.Vertex)) + } + return nil }, ), "load hypergraph", ) + if err != nil { + return nil, err + } + + return hg, nil } func (p *PebbleHypergraphStore) SaveHypergraph( @@ -435,3 +862,316 @@ func (p *PebbleHypergraphStore) SaveHypergraph( return nil } + +func (p *PebbleHypergraphStore) GetNodeByKey( + setType string, + phaseType string, + shardKey crypto.ShardKey, + key []byte, +) (crypto.LazyVectorCommitmentNode, error) { + keyFn := hypergraphVertexAddsTreeNodeKey + switch application.AtomType(setType) { + case application.VertexAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphVertexAddsTreeNodeKey + case application.RemovesPhaseType: + keyFn = hypergraphVertexRemovesTreeNodeKey + } + case application.HyperedgeAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphHyperedgeAddsTreeNodeKey + case application.RemovesPhaseType: + keyFn = hypergraphHyperedgeRemovesTreeNodeKey + } + } + data, closer, err := p.db.Get(keyFn(shardKey, key)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + err = ErrNotFound + return nil, err + } + } + defer closer.Close() + + var node crypto.LazyVectorCommitmentNode + + switch data[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode(p, bytes.NewReader(data[1:])) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(data[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(data[5+(pathLength*4):]), + false, + ) + + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(data[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + return node, errors.Wrap(err, "get node by key") +} + +func (p *PebbleHypergraphStore) GetNodeByPath( + setType string, + phaseType string, + shardKey crypto.ShardKey, + path []int, +) (crypto.LazyVectorCommitmentNode, error) { + keyFn := hypergraphVertexAddsTreeNodeByPathKey + switch application.AtomType(setType) { + case application.VertexAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphVertexAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphVertexRemovesTreeNodeByPathKey + } + case application.HyperedgeAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphHyperedgeAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphHyperedgeRemovesTreeNodeByPathKey + } + } + pathKey := keyFn(shardKey, path) + data, closer, err := p.db.Get(pathKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + err = ErrNotFound + return nil, err + } + } + defer closer.Close() + + nodeData, nodeCloser, err := p.db.Get(data) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + err = ErrNotFound + return nil, err + } + } + defer nodeCloser.Close() + + var node crypto.LazyVectorCommitmentNode + + switch nodeData[0] { + case crypto.TypeLeaf: + node, err = crypto.DeserializeLeafNode( + p, + bytes.NewReader(nodeData[1:]), + ) + case crypto.TypeBranch: + pathLength := binary.BigEndian.Uint32(nodeData[1:5]) + + node, err = crypto.DeserializeBranchNode( + p, + bytes.NewReader(nodeData[5+(pathLength*4):]), + false, + ) + fullPrefix := []int{} + for i := range pathLength { + fullPrefix = append( + fullPrefix, + int(binary.BigEndian.Uint32(nodeData[5+(i*4):5+((i+1)*4)])), + ) + } + branch := node.(*crypto.LazyVectorCommitmentBranchNode) + branch.FullPrefix = fullPrefix + default: + err = ErrInvalidData + } + + return node, errors.Wrap(err, "get node by path") +} + +func generateSlices(fullpref, pref []int) [][]int { + result := [][]int{} + if len(pref) >= len(fullpref) { + return [][]int{fullpref} + } + for i := 0; i <= len(pref); i++ { + newLen := len(fullpref) - i + newSlice := make([]int, newLen) + copy(newSlice, fullpref[:newLen]) + result = append(result, newSlice) + } + + return result +} + +func (p *PebbleHypergraphStore) InsertNode( + setType string, + phaseType string, + shardKey crypto.ShardKey, + key []byte, + path []int, + node crypto.LazyVectorCommitmentNode, +) error { + keyFn := hypergraphVertexAddsTreeNodeKey + pathFn := hypergraphVertexAddsTreeNodeByPathKey + switch application.AtomType(setType) { + case application.VertexAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphVertexAddsTreeNodeKey + pathFn = hypergraphVertexAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphVertexRemovesTreeNodeKey + pathFn = hypergraphVertexRemovesTreeNodeByPathKey + } + case application.HyperedgeAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphHyperedgeAddsTreeNodeKey + pathFn = hypergraphHyperedgeAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphHyperedgeRemovesTreeNodeKey + pathFn = hypergraphHyperedgeRemovesTreeNodeByPathKey + } + } + + var b bytes.Buffer + nodeKey := keyFn(shardKey, key) + switch n := node.(type) { + case *crypto.LazyVectorCommitmentBranchNode: + length := uint32(len(path)) + pathBytes := []byte{} + pathBytes = binary.BigEndian.AppendUint32(pathBytes, length) + for i := range int(length) { + pathBytes = binary.BigEndian.AppendUint32(pathBytes, uint32(path[i])) + } + err := crypto.SerializeBranchNode(&b, n, false) + if err != nil { + return errors.Wrap(err, "insert node") + } + data := append([]byte{crypto.TypeBranch}, pathBytes...) + data = append(data, b.Bytes()...) + err = p.db.Set(nodeKey, data) + if err != nil { + return errors.Wrap(err, "insert node") + } + sets := generateSlices(n.FullPrefix, path) + for _, set := range sets { + pathKey := pathFn(shardKey, set) + err = p.db.Set(pathKey, nodeKey) + if err != nil { + return errors.Wrap(err, "insert node") + } + } + return nil + case *crypto.LazyVectorCommitmentLeafNode: + err := crypto.SerializeLeafNode(&b, n) + if err != nil { + return errors.Wrap(err, "insert node") + } + data := append([]byte{crypto.TypeLeaf}, b.Bytes()...) + pathKey := pathFn(shardKey, path) + err = p.db.Set(nodeKey, data) + if err != nil { + return errors.Wrap(err, "insert node") + } + return errors.Wrap(p.db.Set(pathKey, nodeKey), "insert node") + } + + return nil +} + +func (p *PebbleHypergraphStore) SaveRoot( + setType string, + phaseType string, + shardKey crypto.ShardKey, + node crypto.LazyVectorCommitmentNode, +) error { + keyFn := hypergraphVertexAddsTreeRootKey + switch application.AtomType(setType) { + case application.VertexAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphVertexAddsTreeRootKey + case application.RemovesPhaseType: + keyFn = hypergraphVertexRemovesTreeRootKey + } + case application.HyperedgeAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphHyperedgeAddsTreeRootKey + case application.RemovesPhaseType: + keyFn = hypergraphHyperedgeRemovesTreeRootKey + } + } + + var b bytes.Buffer + nodeKey := keyFn(shardKey) + switch n := node.(type) { + case *crypto.LazyVectorCommitmentBranchNode: + length := uint32(len(n.FullPrefix)) + pathBytes := []byte{} + pathBytes = binary.BigEndian.AppendUint32(pathBytes, length) + for i := range int(length) { + pathBytes = binary.BigEndian.AppendUint32( + pathBytes, + uint32(n.FullPrefix[i]), + ) + } + err := crypto.SerializeBranchNode(&b, n, false) + if err != nil { + return errors.Wrap(err, "insert node") + } + data := append([]byte{crypto.TypeBranch}, pathBytes...) + data = append(data, b.Bytes()...) + err = p.db.Set(nodeKey, data) + return errors.Wrap(err, "insert node") + case *crypto.LazyVectorCommitmentLeafNode: + err := crypto.SerializeLeafNode(&b, n) + if err != nil { + return errors.Wrap(err, "insert node") + } + data := append([]byte{crypto.TypeBranch}, b.Bytes()...) + err = p.db.Set(nodeKey, data) + return errors.Wrap(err, "insert node") + } + + return nil +} + +func (p *PebbleHypergraphStore) DeletePath( + setType string, + phaseType string, + shardKey crypto.ShardKey, + path []int, +) error { + keyFn := hypergraphVertexAddsTreeNodeByPathKey + switch application.AtomType(setType) { + case application.VertexAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphVertexAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphVertexRemovesTreeNodeByPathKey + } + case application.HyperedgeAtomType: + switch application.PhaseType(phaseType) { + case application.AddsPhaseType: + keyFn = hypergraphHyperedgeAddsTreeNodeByPathKey + case application.RemovesPhaseType: + keyFn = hypergraphHyperedgeRemovesTreeNodeByPathKey + } + } + pathKey := keyFn(shardKey, path) + return errors.Wrap(p.db.Delete(pathKey), "delete path") +}