support state tree

node/consensus/data/peer_messaging.go

@@ -458,11 +458,13 @@ func (e *DataClockConsensusEngine) handleMint(
 				return nil, errors.Wrap(err, "handle mint")
 			}
 			returnAddr = proofAddr
+			stateTree := &crypto.VectorCommitmentTree{}
 			err = e.coinStore.PutPreCoinProof(
 				txn,
 				head.FrameNumber,
 				proofAddr,
 				add,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -501,11 +503,13 @@ func (e *DataClockConsensusEngine) handleMint(
 				return nil, errors.Wrap(err, "handle mint")
 			}
 			returnAddr = proofAddr
+			stateTree := &crypto.VectorCommitmentTree{}
 			err = e.coinStore.PutPreCoinProof(
 				txn,
 				head.FrameNumber,
 				proofAddr,
 				proof,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -551,10 +555,12 @@ func (e *DataClockConsensusEngine) handleMint(
 				txn.Abort()
 				return nil, errors.Wrap(err, "handle mint")
 			}
+			stateTree := &crypto.VectorCommitmentTree{}
 			e.coinStore.DeletePreCoinProof(
 				txn,
 				a,
 				deletes[0].GetDeletedProof(),
+				stateTree,
 			)
 		}
 		if err := txn.Commit(); err != nil {

node/consensus/data/token_handle_mint_test.go

@@ -665,31 +665,31 @@ func TestHandlePreMidnightMint(t *testing.T) {
 
 	assert.Len(t, success.Requests, 1)
 	assert.Len(t, fail.Requests, 1)
-
+	stateTree := &qcrypto.VectorCommitmentTree{}
 	txn, _ := app.CoinStore.NewTransaction(false)
 	for i, o := range app.TokenOutputs.Outputs {
 		switch e := o.Output.(type) {
 		case *protobufs.TokenOutput_Coin:
 			a, err := GetAddressOfCoin(e.Coin, 1, uint64(i))
 			assert.NoError(t, err)
-			err = app.CoinStore.PutCoin(txn, 1, a, e.Coin)
+			err = app.CoinStore.PutCoin(txn, 1, a, e.Coin, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_DeletedCoin:
 			c, err := app.CoinStore.GetCoinByAddress(nil, e.DeletedCoin.Address)
 			assert.NoError(t, err)
-			err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c)
+			err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_Proof:
 			a, err := GetAddressOfPreCoinProof(e.Proof)
 			assert.NoError(t, err)
-			err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof)
+			err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_DeletedProof:
 			a, err := GetAddressOfPreCoinProof(e.DeletedProof)
 			assert.NoError(t, err)
 			c, err := app.CoinStore.GetPreCoinProofByAddress(a)
 			assert.NoError(t, err)
-			err = app.CoinStore.DeletePreCoinProof(txn, a, c)
+			err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree)
 			assert.NoError(t, err)
 		}
 	}

node/crypto/proof_tree.go

@@ -0,0 +1,503 @@
+package crypto
+
+import (
+	"bytes"
+	"crypto/sha512"
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	rbls48581 "source.quilibrium.com/quilibrium/monorepo/bls48581"
+)
+
+const (
+	BranchNodes = 1024
+	BranchBits  = 10 // log2(1024)
+	BranchMask  = BranchNodes - 1
+)
+
+type VectorCommitmentNode interface {
+	Commit() []byte
+}
+
+type VectorCommitmentLeafNode struct {
+	key        []byte
+	value      []byte
+	commitment []byte
+}
+
+type VectorCommitmentBranchNode struct {
+	prefix     []int
+	children   [BranchNodes]VectorCommitmentNode
+	commitment []byte
+}
+
+func (n *VectorCommitmentLeafNode) Commit() []byte {
+	if n.commitment == nil {
+		h := sha512.New()
+		h.Write([]byte{0})
+		h.Write(n.key)
+		h.Write(n.value)
+		n.commitment = h.Sum(nil)
+	}
+	return n.commitment
+}
+
+func (n *VectorCommitmentBranchNode) Commit() []byte {
+	if n.commitment == nil {
+		data := []byte{}
+		for _, child := range n.children {
+			if child != nil {
+				out := child.Commit()
+				switch c := child.(type) {
+				case *VectorCommitmentBranchNode:
+					h := sha512.New()
+					h.Write([]byte{1})
+					for _, p := range c.prefix {
+						h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
+					}
+					h.Write(out)
+					out = h.Sum(nil)
+				case *VectorCommitmentLeafNode:
+					// do nothing
+				}
+				data = append(data, out...)
+			} else {
+				data = append(data, make([]byte, 64)...)
+			}
+		}
+
+		n.commitment = rbls48581.CommitRaw(data, 1024)
+	}
+
+	return n.commitment
+}
+
+func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool {
+	data := []byte{}
+	if n.commitment == nil {
+		for _, child := range n.children {
+			if child != nil {
+				out := child.Commit()
+				switch c := child.(type) {
+				case *VectorCommitmentBranchNode:
+					h := sha512.New()
+					h.Write([]byte{1})
+					for _, p := range c.prefix {
+						h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
+					}
+					h.Write(out)
+					out = h.Sum(nil)
+				case *VectorCommitmentLeafNode:
+					// do nothing
+				}
+				data = append(data, out...)
+			} else {
+				data = append(data, make([]byte, 64)...)
+			}
+		}
+
+		n.commitment = rbls48581.CommitRaw(data, 1024)
+		data = data[64*index : 64*(index+1)]
+	} else {
+		child := n.children[index]
+		if child != nil {
+			out := child.Commit()
+			switch c := child.(type) {
+			case *VectorCommitmentBranchNode:
+				h := sha512.New()
+				h.Write([]byte{1})
+				for _, p := range c.prefix {
+					h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
+				}
+				h.Write(out)
+				out = h.Sum(nil)
+			case *VectorCommitmentLeafNode:
+				// do nothing
+			}
+			data = append(data, out...)
+		} else {
+			data = append(data, make([]byte, 64)...)
+		}
+	}
+
+	return rbls48581.VerifyRaw(data, n.commitment, uint64(index), proof, 1024)
+}
+
+func (n *VectorCommitmentBranchNode) Prove(index int) []byte {
+	data := []byte{}
+	for _, child := range n.children {
+		if child != nil {
+			out := child.Commit()
+			switch c := child.(type) {
+			case *VectorCommitmentBranchNode:
+				h := sha512.New()
+				h.Write([]byte{1})
+				for _, p := range c.prefix {
+					h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
+				}
+				h.Write(out)
+				out = h.Sum(nil)
+			case *VectorCommitmentLeafNode:
+				// do nothing
+			}
+			data = append(data, out...)
+		} else {
+			data = append(data, make([]byte, 64)...)
+		}
+	}
+
+	return rbls48581.ProveRaw(data, uint64(index), 1024)
+}
+
+type VectorCommitmentTree struct {
+	root VectorCommitmentNode
+}
+
+// getNextNibble returns the next BranchBits bits from the key starting at pos
+func getNextNibble(key []byte, pos int) int {
+	startByte := pos / 8
+	if startByte >= len(key) {
+		return 0
+	}
+
+	// Calculate how many bits we need from the current byte
+	startBit := pos % 8
+	bitsFromCurrentByte := 8 - startBit
+
+	result := int(key[startByte] & ((1 << bitsFromCurrentByte) - 1))
+
+	if bitsFromCurrentByte >= BranchBits {
+		// We have enough bits in the current byte
+		return (result >> (bitsFromCurrentByte - BranchBits)) & BranchMask
+	}
+
+	// We need bits from the next byte
+	result = result << (BranchBits - bitsFromCurrentByte)
+	if startByte+1 < len(key) {
+		remainingBits := BranchBits - bitsFromCurrentByte
+		nextByte := int(key[startByte+1])
+		result |= (nextByte >> (8 - remainingBits))
+	}
+
+	return result & BranchMask
+}
+
+func getNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) {
+	var nibbles []int
+	depth := startDepth
+
+	for {
+		n1 := getNextNibble(key1, depth)
+		n2 := getNextNibble(key2, depth)
+		if n1 != n2 {
+			return nibbles, depth
+		}
+		nibbles = append(nibbles, n1)
+		depth += BranchBits
+	}
+}
+
+// getLastNibble returns the final nibble after applying a prefix
+func getLastNibble(key []byte, prefixLen int) int {
+	return getNextNibble(key, prefixLen*BranchBits)
+}
+
+// Insert adds or updates a key-value pair in the tree
+func (t *VectorCommitmentTree) Insert(key, value []byte) error {
+	if len(key) == 0 {
+		return errors.New("empty key not allowed")
+	}
+
+	var insert func(node VectorCommitmentNode, depth int) VectorCommitmentNode
+	insert = func(node VectorCommitmentNode, depth int) VectorCommitmentNode {
+		if node == nil {
+			return &VectorCommitmentLeafNode{key: key, value: value}
+		}
+
+		switch n := node.(type) {
+		case *VectorCommitmentLeafNode:
+			if bytes.Equal(n.key, key) {
+				n.value = value
+				n.commitment = nil
+				return n
+			}
+
+			// Get common prefix nibbles and divergence point
+			sharedNibbles, divergeDepth := getNibblesUntilDiverge(n.key, key, depth)
+
+			// Create single branch node with shared prefix
+			branch := &VectorCommitmentBranchNode{
+				prefix: sharedNibbles,
+			}
+
+			// Add both leaves at their final positions
+			finalOldNibble := getNextNibble(n.key, divergeDepth)
+			finalNewNibble := getNextNibble(key, divergeDepth)
+			branch.children[finalOldNibble] = n
+			branch.children[finalNewNibble] = &VectorCommitmentLeafNode{key: key, value: value}
+
+			return branch
+
+		case *VectorCommitmentBranchNode:
+			if len(n.prefix) > 0 {
+				// Check if the new key matches the prefix
+				for i, expectedNibble := range n.prefix {
+					actualNibble := getNextNibble(key, depth+i*BranchBits)
+					if actualNibble != expectedNibble {
+						// Create new branch with shared prefix subset
+						newBranch := &VectorCommitmentBranchNode{
+							prefix: n.prefix[:i],
+						}
+						// Position old branch and new leaf
+						newBranch.children[expectedNibble] = n
+						n.prefix = n.prefix[i+1:] // remove shared prefix from old branch
+						newBranch.children[actualNibble] = &VectorCommitmentLeafNode{key: key, value: value}
+						return newBranch
+					}
+				}
+				// Key matches prefix, continue with final nibble
+				finalNibble := getNextNibble(key, depth+len(n.prefix)*BranchBits)
+				n.children[finalNibble] = insert(n.children[finalNibble], depth+len(n.prefix)*BranchBits+BranchBits)
+				n.commitment = nil
+				return n
+			} else {
+				// Simple branch without prefix
+				nibble := getNextNibble(key, depth)
+				n.children[nibble] = insert(n.children[nibble], depth+BranchBits)
+				n.commitment = nil
+				return n
+			}
+		}
+
+		return nil
+	}
+
+	t.root = insert(t.root, 0)
+	return nil
+}
+
+func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
+	if len(key) == 0 {
+		return false
+	}
+
+	var verify func(node VectorCommitmentNode, proofs [][]byte, depth int) bool
+	verify = func(node VectorCommitmentNode, proofs [][]byte, depth int) bool {
+		if node == nil {
+			return false
+		}
+
+		if len(proofs) == 0 {
+			return false
+		}
+
+		switch n := node.(type) {
+		case *VectorCommitmentLeafNode:
+			if bytes.Equal(n.key, key) {
+				return bytes.Equal(n.value, proofs[0])
+			}
+			return false
+
+		case *VectorCommitmentBranchNode:
+			// Check prefix match
+			for i, expectedNibble := range n.prefix {
+				if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
+					return false
+				}
+			}
+
+			// Get final nibble after prefix
+			finalNibble := getNextNibble(key, depth+len(n.prefix)*BranchBits)
+
+			if !n.Verify(finalNibble, proofs[0]) {
+				return false
+			}
+
+			return verify(n.children[finalNibble], proofs[1:], depth+len(n.prefix)*BranchBits+BranchBits)
+		}
+
+		return false
+	}
+
+	return verify(t.root, proofs, 0)
+}
+
+func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
+	if len(key) == 0 {
+		return nil
+	}
+
+	var prove func(node VectorCommitmentNode, depth int) [][]byte
+	prove = func(node VectorCommitmentNode, depth int) [][]byte {
+		if node == nil {
+			return nil
+		}
+
+		switch n := node.(type) {
+		case *VectorCommitmentLeafNode:
+			if bytes.Equal(n.key, key) {
+				return [][]byte{n.value}
+			}
+			return nil
+
+		case *VectorCommitmentBranchNode:
+			// Check prefix match
+			for i, expectedNibble := range n.prefix {
+				if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
+					return nil
+				}
+			}
+
+			// Get final nibble after prefix
+			finalNibble := getNextNibble(key, depth+len(n.prefix)*BranchBits)
+
+			proofs := [][]byte{n.Prove(finalNibble)}
+
+			return append(proofs, prove(n.children[finalNibble], depth+len(n.prefix)*BranchBits+BranchBits)...)
+		}
+
+		return nil
+	}
+
+	return prove(t.root, 0)
+}
+
+// Get retrieves a value from the tree by key
+func (t *VectorCommitmentTree) Get(key []byte) ([]byte, error) {
+	if len(key) == 0 {
+		return nil, errors.New("empty key not allowed")
+	}
+
+	var get func(node VectorCommitmentNode, depth int) []byte
+	get = func(node VectorCommitmentNode, depth int) []byte {
+		if node == nil {
+			return nil
+		}
+
+		switch n := node.(type) {
+		case *VectorCommitmentLeafNode:
+			if bytes.Equal(n.key, key) {
+				return n.value
+			}
+			return nil
+
+		case *VectorCommitmentBranchNode:
+			// Check prefix match
+			for i, expectedNibble := range n.prefix {
+				if getNextNibble(key, depth+i*BranchBits) != expectedNibble {
+					return nil
+				}
+			}
+			// Get final nibble after prefix
+			finalNibble := getNextNibble(key, depth+len(n.prefix)*BranchBits)
+			return get(n.children[finalNibble], depth+len(n.prefix)*BranchBits+BranchBits)
+		}
+
+		return nil
+	}
+
+	value := get(t.root, 0)
+	if value == nil {
+		return nil, errors.New("key not found")
+	}
+	return value, nil
+}
+
+// Delete removes a key-value pair from the tree
+func (t *VectorCommitmentTree) Delete(key []byte) error {
+	if len(key) == 0 {
+		return errors.New("empty key not allowed")
+	}
+
+	var delete func(node VectorCommitmentNode, depth int) VectorCommitmentNode
+	delete = func(node VectorCommitmentNode, depth int) VectorCommitmentNode {
+		if node == nil {
+			return nil
+		}
+
+		switch n := node.(type) {
+		case *VectorCommitmentLeafNode:
+			if bytes.Equal(n.key, key) {
+				return nil
+			}
+			return n
+
+		case *VectorCommitmentBranchNode:
+			// Check prefix match
+			for i, expectedNibble := range n.prefix {
+				currentNibble := getNextNibble(key, depth+i*BranchBits)
+				if currentNibble != expectedNibble {
+					return n // Key doesn't match prefix, nothing to delete
+				}
+			}
+
+			// Delete at final position after prefix
+			finalNibble := getNextNibble(key, depth+len(n.prefix)*BranchBits)
+			n.children[finalNibble] = delete(n.children[finalNibble], depth+len(n.prefix)*BranchBits+BranchBits)
+			n.commitment = nil
+
+			// Count remaining children
+			childCount := 0
+			var lastChild VectorCommitmentNode
+			var lastIndex int
+			for i, child := range n.children {
+				if child != nil {
+					childCount++
+					lastChild = child
+					lastIndex = i
+				}
+			}
+
+			if childCount == 0 {
+				return nil
+			} else if childCount == 1 {
+				// If the only child is a leaf, keep structure if its path matches
+				if leaf, ok := lastChild.(*VectorCommitmentLeafNode); ok {
+					if lastIndex == getLastNibble(leaf.key, len(n.prefix)) {
+						return n
+					}
+					return leaf
+				}
+				// If it's a branch, merge the prefixes
+				if branch, ok := lastChild.(*VectorCommitmentBranchNode); ok {
+					branch.prefix = append(n.prefix, branch.prefix...)
+					return branch
+				}
+			}
+			return n
+		}
+
+		return nil
+	}
+
+	t.root = delete(t.root, 0)
+	return nil
+}
+
+// Root returns the root hash of the tree
+func (t *VectorCommitmentTree) Root() []byte {
+	if t.root == nil {
+		return make([]byte, 64)
+	}
+	return t.root.Commit()
+}
+
+func debugNode(node VectorCommitmentNode, depth int, prefix string) {
+	if node == nil {
+		return
+	}
+
+	switch n := node.(type) {
+	case *VectorCommitmentLeafNode:
+		fmt.Printf("%sLeaf: key=%x value=%x\n", prefix, n.key, n.value)
+	case *VectorCommitmentBranchNode:
+		fmt.Printf("%sBranch %v:\n", prefix, n.prefix)
+		for i, child := range n.children {
+			if child != nil {
+				fmt.Printf("%s  [%d]:\n", prefix, i)
+				debugNode(child, depth+1, prefix+"    ")
+			}
+		}
+	}
+}

node/crypto/proof_tree_test.go

@@ -0,0 +1,221 @@
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"testing"
+
+	"source.quilibrium.com/quilibrium/monorepo/bls48581/generated/bls48581"
+)
+
+func TestVectorCommitmentTrees(t *testing.T) {
+	bls48581.Init()
+	tree := &VectorCommitmentTree{}
+
+	// Test single insert
+	err := tree.Insert([]byte("key1"), []byte("value1"))
+	if err != nil {
+		t.Errorf("Failed to insert: %v", err)
+	}
+
+	// Test duplicate key
+	err = tree.Insert([]byte("key1"), []byte("value2"))
+	if err != nil {
+		t.Errorf("Failed to update existing key: %v", err)
+	}
+
+	value, err := tree.Get([]byte("key1"))
+	if err != nil {
+		t.Errorf("Failed to get value: %v", err)
+	}
+	if !bytes.Equal(value, []byte("value2")) {
+		t.Errorf("Expected value2, got %s", string(value))
+	}
+
+	// Test empty key
+	err = tree.Insert([]byte{}, []byte("value"))
+	if err == nil {
+		t.Error("Expected error for empty key, got none")
+	}
+
+	tree = &VectorCommitmentTree{}
+
+	// Test get on empty tree
+	_, err = tree.Get([]byte("nonexistent"))
+	if err == nil {
+		t.Error("Expected error for nonexistent key, got none")
+	}
+
+	// Insert and get
+	tree.Insert([]byte("key1"), []byte("value1"))
+	value, err = tree.Get([]byte("key1"))
+	if err != nil {
+		t.Errorf("Failed to get value: %v", err)
+	}
+	if !bytes.Equal(value, []byte("value1")) {
+		t.Errorf("Expected value1, got %s", string(value))
+	}
+
+	// Test empty key
+	_, err = tree.Get([]byte{})
+	if err == nil {
+		t.Error("Expected error for empty key, got none")
+	}
+
+	tree = &VectorCommitmentTree{}
+
+	// Test delete on empty tree
+	err = tree.Delete([]byte("nonexistent"))
+	if err != nil {
+		t.Errorf("Delete on empty tree should not return error: %v", err)
+	}
+
+	// Insert and delete
+	tree.Insert([]byte("key1"), []byte("value1"))
+	err = tree.Delete([]byte("key1"))
+	if err != nil {
+		t.Errorf("Failed to delete: %v", err)
+	}
+
+	// Verify deletion
+	_, err = tree.Get([]byte("key1"))
+	if err == nil {
+		t.Error("Expected error for deleted key, got none")
+	}
+
+	// Test empty key
+	err = tree.Delete([]byte{})
+	if err == nil {
+		t.Error("Expected error for empty key, got none")
+	}
+
+	tree = &VectorCommitmentTree{}
+
+	// Insert keys that share common prefix
+	keys := []string{
+		"key1",
+		"key2",
+		"key3",
+		"completely_different",
+	}
+
+	for i, key := range keys {
+		err := tree.Insert([]byte(key), []byte("value"+string(rune('1'+i))))
+		if err != nil {
+			t.Errorf("Failed to insert key %s: %v", key, err)
+		}
+	}
+
+	// Verify all values
+	for i, key := range keys {
+		value, err := tree.Get([]byte(key))
+		if err != nil {
+			t.Errorf("Failed to get key %s: %v", key, err)
+		}
+		expected := []byte("value" + string(rune('1'+i)))
+		if !bytes.Equal(value, expected) {
+			t.Errorf("Expected %s, got %s", string(expected), string(value))
+		}
+	}
+
+	// Delete middle key
+	err = tree.Delete([]byte("key2"))
+	if err != nil {
+		t.Errorf("Failed to delete key2: %v", err)
+	}
+
+	// Verify key2 is gone but others remain
+	_, err = tree.Get([]byte("key2"))
+	if err == nil {
+		t.Error("Expected error for deleted key2, got none")
+	}
+
+	// Check remaining keys
+	remainingKeys := []string{"key1", "key3", "completely_different"}
+	remainingValues := []string{"value1", "value3", "value4"}
+	for i, key := range remainingKeys {
+		value, err := tree.Get([]byte(key))
+		if err != nil {
+			t.Errorf("Failed to get key %s after deletion: %v", key, err)
+		}
+		expected := []byte(remainingValues[i])
+		if !bytes.Equal(value, expected) {
+			t.Errorf("Expected %s, got %s", string(expected), string(value))
+		}
+	}
+
+	tree = &VectorCommitmentTree{}
+
+	// Empty tree should have zero hash
+	emptyRoot := tree.Root()
+	if len(emptyRoot) != 64 {
+		t.Errorf("Expected 64 byte root hash, got %d bytes", len(emptyRoot))
+	}
+
+	// Root should change after insert
+	tree.Insert([]byte("key1"), []byte("value1"))
+	firstRoot := tree.Root()
+
+	if bytes.Equal(firstRoot, emptyRoot) {
+		t.Error("Root hash should change after insert")
+	}
+
+	// Root should change after update
+	tree.Insert([]byte("key1"), []byte("value2"))
+	secondRoot := tree.Root()
+
+	if bytes.Equal(secondRoot, firstRoot) {
+		t.Error("Root hash should change after update")
+	}
+
+	// Root should change after delete
+	tree.Delete([]byte("key1"))
+	thirdRoot := tree.Root()
+
+	if !bytes.Equal(thirdRoot, emptyRoot) {
+		t.Error("Root hash should match empty tree after deleting all entries")
+	}
+
+	tree = &VectorCommitmentTree{}
+
+	addresses := [][]byte{}
+
+	for i := 0; i < 1000; i++ {
+		d := make([]byte, 32)
+		rand.Read(d)
+		addresses = append(addresses, d)
+	}
+
+	// Insert 100 items
+	for i := 0; i < 1000; i++ {
+		key := addresses[i]
+		value := addresses[i]
+		err := tree.Insert(key, value)
+		if err != nil {
+			t.Errorf("Failed to insert item %d: %v", i, err)
+		}
+	}
+
+	// Verify all items
+	for i := 0; i < 1000; i++ {
+		key := addresses[i]
+		expected := addresses[i]
+		value, err := tree.Get(key)
+		if err != nil {
+			t.Errorf("Failed to get item %d: %v", i, err)
+		}
+		if !bytes.Equal(value, expected) {
+			t.Errorf("Item %d: expected %x, got %x", i, string(expected), string(value))
+		}
+	}
+
+	proofs := tree.Prove(addresses[500])
+	if !tree.Verify(addresses[500], proofs) {
+		t.Errorf("proof failed")
+	}
+
+	for _, p := range proofs {
+		fmt.Printf("%x\n", p)
+	}
+}

node/execution/intrinsics/token/application/token_handle_prover_join_test.go

@@ -181,29 +181,30 @@ func TestHandleProverJoin(t *testing.T) {
 	assert.Len(t, success.Requests, 1)
 	assert.Len(t, app.TokenOutputs.Outputs, 1)
 	txn, _ = app.CoinStore.NewTransaction(false)
+	stateTree := &qcrypto.VectorCommitmentTree{}
 	for i, o := range app.TokenOutputs.Outputs {
 		switch e := o.Output.(type) {
 		case *protobufs.TokenOutput_Coin:
 			a, err := token.GetAddressOfCoin(e.Coin, 1, uint64(i))
 			assert.NoError(t, err)
-			err = app.CoinStore.PutCoin(txn, 1, a, e.Coin)
+			err = app.CoinStore.PutCoin(txn, 1, a, e.Coin, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_DeletedCoin:
 			c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address)
 			assert.NoError(t, err)
-			err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c)
+			err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_Proof:
 			a, err := token.GetAddressOfPreCoinProof(e.Proof)
 			assert.NoError(t, err)
-			err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof)
+			err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree)
 			assert.NoError(t, err)
 		case *protobufs.TokenOutput_DeletedProof:
 			a, err := token.GetAddressOfPreCoinProof(e.DeletedProof)
 			assert.NoError(t, err)
 			c, err := app.CoinStore.GetPreCoinProofByAddress(a)
 			assert.NoError(t, err)
-			err = app.CoinStore.DeletePreCoinProof(txn, a, c)
+			err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree)
 			assert.NoError(t, err)
 		}
 	}

node/execution/intrinsics/token/token_execution_engine.go

@@ -104,6 +104,7 @@ type TokenExecutionEngine struct {
 	intrinsicFilter       []byte
 	frameProver           qcrypto.FrameProver
 	peerSeniority         *PeerSeniority
+	stateTree             *qcrypto.VectorCommitmentTree
 }
 
 func NewTokenExecutionEngine(
@@ -137,6 +138,7 @@ func NewTokenExecutionEngine(
 	var inclusionProof *qcrypto.InclusionAggregateProof
 	var proverKeys [][]byte
 	var peerSeniority map[string]uint64
+	stateTree := &qcrypto.VectorCommitmentTree{}
 
 	if err != nil && errors.Is(err, store.ErrNotFound) {
 		origin, inclusionProof, proverKeys, peerSeniority = CreateGenesisState(
@@ -146,6 +148,7 @@ func NewTokenExecutionEngine(
 			inclusionProver,
 			clockStore,
 			coinStore,
+			stateTree,
 			uint(cfg.P2P.Network),
 		)
 		if err := coinStore.SetMigrationVersion(
@@ -172,6 +175,7 @@ func NewTokenExecutionEngine(
 				inclusionProver,
 				clockStore,
 				coinStore,
+				stateTree,
 				uint(cfg.P2P.Network),
 			)
 		}
@@ -342,6 +346,15 @@ func NewTokenExecutionEngine(
 	e.proverPublicKey = publicKeyBytes
 	e.provingKeyAddress = provingKeyAddress
 
+	e.stateTree, err = e.clockStore.GetDataStateTree(e.intrinsicFilter)
+	if err != nil && !errors.Is(err, store.ErrNotFound) {
+		panic(err)
+	}
+
+	if e.stateTree == nil {
+		e.rebuildStateTree()
+	}
+
 	e.wg.Add(1)
 	go func() {
 		defer e.wg.Done()
@@ -406,6 +419,45 @@ func NewTokenExecutionEngine(
 
 var _ execution.ExecutionEngine = (*TokenExecutionEngine)(nil)
 
+func (e *TokenExecutionEngine) rebuildStateTree() {
+	e.logger.Info("rebuilding state tree")
+	e.stateTree = &qcrypto.VectorCommitmentTree{}
+	iter, err := e.coinStore.RangeCoins()
+	if err != nil {
+		panic(err)
+	}
+	for iter.First(); iter.Valid(); iter.Next() {
+		e.stateTree.Insert(iter.Key()[2:], iter.Value())
+	}
+	iter.Close()
+
+	iter, err = e.coinStore.RangePreCoinProofs()
+	if err != nil {
+		panic(err)
+	}
+	for iter.First(); iter.Valid(); iter.Next() {
+		e.stateTree.Insert(iter.Key()[2:], iter.Value())
+	}
+	iter.Close()
+	e.logger.Info("saving rebuilt state tree")
+
+	txn, err := e.clockStore.NewTransaction(false)
+	if err != nil {
+		panic(err)
+	}
+
+	err = e.clockStore.SetDataStateTree(txn, e.intrinsicFilter, e.stateTree)
+	if err != nil {
+		txn.Abort()
+		panic(err)
+	}
+
+	if err = txn.Commit(); err != nil {
+		txn.Abort()
+		panic(err)
+	}
+}
+
 // GetName implements ExecutionEngine
 func (*TokenExecutionEngine) GetName() string {
 	return "Token"
@@ -571,6 +623,12 @@ func (e *TokenExecutionEngine) ProcessFrame(
 	}
 	wg.Wait()
 
+	stateTree, err := e.clockStore.GetDataStateTree(e.intrinsicFilter)
+	if err != nil {
+		txn.Abort()
+		return nil, errors.Wrap(err, "process frame")
+	}
+
 	for i, output := range app.TokenOutputs.Outputs {
 		switch o := output.Output.(type) {
 		case *protobufs.TokenOutput_Coin:
@@ -584,6 +642,7 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				frame.FrameNumber,
 				address,
 				o.Coin,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -599,6 +658,7 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				txn,
 				o.DeletedCoin.Address,
 				coin,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -615,6 +675,7 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				frame.FrameNumber,
 				address,
 				o.Proof,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -652,6 +713,7 @@ func (e *TokenExecutionEngine) ProcessFrame(
 				txn,
 				address,
 				o.DeletedProof,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -968,6 +1030,18 @@ func (e *TokenExecutionEngine) ProcessFrame(
 		}
 	}
 
+	err = e.clockStore.SetDataStateTree(
+		txn,
+		e.intrinsicFilter,
+		stateTree,
+	)
+	if err != nil {
+		txn.Abort()
+		return nil, errors.Wrap(err, "process frame")
+	}
+
+	e.stateTree = stateTree
+
 	return app.Tries, nil
 }
 

node/execution/intrinsics/token/token_genesis.go

@@ -17,6 +17,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"source.quilibrium.com/quilibrium/monorepo/nekryptology/pkg/vdf"
 	"source.quilibrium.com/quilibrium/monorepo/node/config"
+	"source.quilibrium.com/quilibrium/monorepo/node/crypto"
 	qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto"
 	"source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application"
 	"source.quilibrium.com/quilibrium/monorepo/node/p2p"
@@ -503,6 +504,7 @@ func CreateGenesisState(
 	inclusionProver qcrypto.InclusionProver,
 	clockStore store.ClockStore,
 	coinStore store.CoinStore,
+	stateTree *crypto.VectorCommitmentTree,
 	network uint,
 ) (
 	[]byte,
@@ -863,6 +865,7 @@ func CreateGenesisState(
 				0,
 				address,
 				output.GetCoin(),
+				stateTree,
 			)
 			if err != nil {
 				panic(err)
@@ -886,6 +889,13 @@ func CreateGenesisState(
 			panic(err)
 		}
 
+		intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3)
+		err = clockStore.SetDataStateTree(txn, intrinsicFilter, stateTree)
+		if err != nil {
+			txn.Abort()
+			panic(err)
+		}
+
 		if err = txn.Commit(); err != nil {
 			panic(err)
 		}
@@ -897,8 +907,6 @@ func CreateGenesisState(
 			panic(err)
 		}
 
-		intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3)
-
 		executionOutput := &protobufs.IntrinsicExecutionOutput{
 			Address: intrinsicFilter,
 			Output:  outputBytes,
@@ -1004,11 +1012,19 @@ func CreateGenesisState(
 				0,
 				address,
 				output.GetCoin(),
+				stateTree,
 			)
 			if err != nil {
 				panic(err)
 			}
 		}
+		intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3)
+		err = clockStore.SetDataStateTree(txn, intrinsicFilter, stateTree)
+		if err != nil {
+			txn.Abort()
+			panic(err)
+		}
+
 		if err := txn.Commit(); err != nil {
 			panic(err)
 		}
@@ -1020,8 +1036,6 @@ func CreateGenesisState(
 			panic(err)
 		}
 
-		intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3)
-
 		executionOutput := &protobufs.IntrinsicExecutionOutput{
 			Address: intrinsicFilter,
 			Output:  outputBytes,

node/main.go

@@ -633,7 +633,8 @@ func RunForkRepairIfNeeded(
 					fmt.Println(err)
 					return
 				}
-				if err = coinStore.DeleteCoin(txn, address, coin); err != nil {
+				stateTree := &qcrypto.VectorCommitmentTree{}
+				if err = coinStore.DeleteCoin(txn, address, coin, stateTree); err != nil {
 					txn.Abort()
 					fmt.Println(err)
 					return
@@ -646,7 +647,8 @@ func RunForkRepairIfNeeded(
 					fmt.Println(err)
 					return
 				}
-				if err = coinStore.DeletePreCoinProof(txn, address, proof); err != nil {
+				stateTree := &qcrypto.VectorCommitmentTree{}
+				if err = coinStore.DeletePreCoinProof(txn, address, proof, stateTree); err != nil {
 					txn.Abort()
 					fmt.Println(err)
 					return
@@ -819,11 +821,13 @@ func processFrame(
 				txn.Abort()
 				return nil, errors.Wrap(err, "process frame")
 			}
+			stateTree := &qcrypto.VectorCommitmentTree{}
 			err = coinStore.PutCoin(
 				txn,
 				frame.FrameNumber,
 				address,
 				o.Coin,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -839,10 +843,12 @@ func processFrame(
 				txn.Abort()
 				return nil, errors.Wrap(err, "process frame")
 			}
+			stateTree := &qcrypto.VectorCommitmentTree{}
 			err = coinStore.DeleteCoin(
 				txn,
 				o.DeletedCoin.Address,
 				coin,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -854,11 +860,13 @@ func processFrame(
 				txn.Abort()
 				return nil, errors.Wrap(err, "process frame")
 			}
+			stateTree := &qcrypto.VectorCommitmentTree{}
 			err = coinStore.PutPreCoinProof(
 				txn,
 				frame.FrameNumber,
 				address,
 				o.Proof,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()
@@ -889,10 +897,12 @@ func processFrame(
 				txn.Abort()
 				return nil, errors.Wrap(err, "process frame")
 			}
+			stateTree := &qcrypto.VectorCommitmentTree{}
 			err = coinStore.DeletePreCoinProof(
 				txn,
 				address,
 				o.DeletedProof,
+				stateTree,
 			)
 			if err != nil {
 				txn.Abort()

node/store/clock.go

@@ -13,6 +13,7 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/proto"
 	"source.quilibrium.com/quilibrium/monorepo/node/config"
+	"source.quilibrium.com/quilibrium/monorepo/node/crypto"
 	"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
 	"source.quilibrium.com/quilibrium/monorepo/node/tries"
 )
@@ -103,6 +104,12 @@ type ClockStore interface {
 		minFrameNumber uint64,
 		maxFrameNumber uint64,
 	) error
+	GetDataStateTree(filter []byte) (*crypto.VectorCommitmentTree, error)
+	SetDataStateTree(
+		txn Transaction,
+		filter []byte,
+		tree *crypto.VectorCommitmentTree,
+	) error
 }
 
 type PebbleClockStore struct {
@@ -298,6 +305,7 @@ const CLOCK_DATA_FRAME_FRECENCY_DATA = 0x03
 const CLOCK_DATA_FRAME_DISTANCE_DATA = 0x04
 const CLOCK_COMPACTION_DATA = 0x05
 const CLOCK_DATA_FRAME_SENIORITY_DATA = 0x06
+const CLOCK_DATA_FRAME_STATE_TREE = 0x07
 const CLOCK_MASTER_FRAME_INDEX_EARLIEST = 0x10 | CLOCK_MASTER_FRAME_DATA
 const CLOCK_MASTER_FRAME_INDEX_LATEST = 0x20 | CLOCK_MASTER_FRAME_DATA
 const CLOCK_MASTER_FRAME_INDEX_PARENT = 0x30 | CLOCK_MASTER_FRAME_DATA
@@ -453,6 +461,14 @@ func clockDataSeniorityKey(
 	return key
 }
 
+func clockDataStateTreeKey(
+	filter []byte,
+) []byte {
+	key := []byte{CLOCK_FRAME, CLOCK_DATA_FRAME_STATE_TREE}
+	key = append(key, filter...)
+	return key
+}
+
 func (p *PebbleClockStore) NewTransaction(indexed bool) (Transaction, error) {
 	return p.db.NewBatch(indexed), nil
 }
@@ -1627,3 +1643,45 @@ func (p *PebbleClockStore) SetProverTriesForFrame(
 
 	return nil
 }
+
+func (p *PebbleClockStore) GetDataStateTree(filter []byte) (
+	*crypto.VectorCommitmentTree,
+	error,
+) {
+	data, closer, err := p.db.Get(clockDataStateTreeKey(filter))
+	if err != nil {
+		if errors.Is(err, pebble.ErrNotFound) {
+			return nil, ErrNotFound
+		}
+
+		return nil, errors.Wrap(err, "get data state tree")
+	}
+	defer closer.Close()
+	tree := &crypto.VectorCommitmentTree{}
+	var b bytes.Buffer
+	b.Write(data)
+	dec := gob.NewDecoder(&b)
+	if err = dec.Decode(tree); err != nil {
+		return nil, errors.Wrap(err, "get data state tree")
+	}
+
+	return tree, nil
+}
+
+func (p *PebbleClockStore) SetDataStateTree(
+	txn Transaction,
+	filter []byte,
+	tree *crypto.VectorCommitmentTree,
+) error {
+	b := new(bytes.Buffer)
+	enc := gob.NewEncoder(b)
+
+	if err := enc.Encode(tree); err != nil {
+		return errors.Wrap(err, "set data state tree")
+	}
+
+	return errors.Wrap(
+		txn.Set(clockDataStateTreeKey(filter), b.Bytes()),
+		"set data state tree",
+	)
+}

node/store/coin.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/proto"
+	"source.quilibrium.com/quilibrium/monorepo/node/crypto"
 	"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
 )
 
@@ -23,28 +24,33 @@ type CoinStore interface {
 	)
 	GetCoinByAddress(txn Transaction, address []byte) (*protobufs.Coin, error)
 	GetPreCoinProofByAddress(address []byte) (*protobufs.PreCoinProof, error)
+	RangeCoins() (Iterator, error)
 	RangePreCoinProofs() (Iterator, error)
 	PutCoin(
 		txn Transaction,
 		frameNumber uint64,
 		address []byte,
 		coin *protobufs.Coin,
+		stateTree *crypto.VectorCommitmentTree,
 	) error
 	DeleteCoin(
 		txn Transaction,
 		address []byte,
 		coin *protobufs.Coin,
+		stateTree *crypto.VectorCommitmentTree,
 	) error
 	PutPreCoinProof(
 		txn Transaction,
 		frameNumber uint64,
 		address []byte,
 		preCoinProof *protobufs.PreCoinProof,
+		stateTree *crypto.VectorCommitmentTree,
 	) error
 	DeletePreCoinProof(
 		txn Transaction,
 		address []byte,
 		preCoinProof *protobufs.PreCoinProof,
+		stateTree *crypto.VectorCommitmentTree,
 	) error
 	GetLatestFrameProcessed() (uint64, error)
 	SetLatestFrameProcessed(txn Transaction, frameNumber uint64) error
@@ -264,11 +270,24 @@ func (p *PebbleCoinStore) RangePreCoinProofs() (Iterator, error) {
 	return iter, nil
 }
 
+func (p *PebbleCoinStore) RangeCoins() (Iterator, error) {
+	iter, err := p.db.NewIter(
+		coinKey(bytes.Repeat([]byte{0x00}, 32)),
+		coinKey(bytes.Repeat([]byte{0xff}, 32)),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "range pre coin proofs")
+	}
+
+	return iter, nil
+}
+
 func (p *PebbleCoinStore) PutCoin(
 	txn Transaction,
 	frameNumber uint64,
 	address []byte,
 	coin *protobufs.Coin,
+	stateTree *crypto.VectorCommitmentTree,
 ) error {
 	coinBytes, err := proto.Marshal(coin)
 	if err != nil {
@@ -294,6 +313,10 @@ func (p *PebbleCoinStore) PutCoin(
 		return errors.Wrap(err, "put coin")
 	}
 
+	if err = stateTree.Insert(address, data); err != nil {
+		return errors.Wrap(err, "put coin")
+	}
+
 	return nil
 }
 
@@ -301,6 +324,7 @@ func (p *PebbleCoinStore) DeleteCoin(
 	txn Transaction,
 	address []byte,
 	coin *protobufs.Coin,
+	stateTree *crypto.VectorCommitmentTree,
 ) error {
 	err := txn.Delete(coinKey(address))
 	if err != nil {
@@ -314,6 +338,10 @@ func (p *PebbleCoinStore) DeleteCoin(
 		return errors.Wrap(err, "delete coin")
 	}
 
+	if err = stateTree.Delete(address); err != nil {
+		return errors.Wrap(err, "delete coin")
+	}
+
 	return nil
 }
 
@@ -322,6 +350,7 @@ func (p *PebbleCoinStore) PutPreCoinProof(
 	frameNumber uint64,
 	address []byte,
 	preCoinProof *protobufs.PreCoinProof,
+	stateTree *crypto.VectorCommitmentTree,
 ) error {
 	proofBytes, err := proto.Marshal(preCoinProof)
 	if err != nil {
@@ -347,6 +376,10 @@ func (p *PebbleCoinStore) PutPreCoinProof(
 		return errors.Wrap(err, "put pre coin proof")
 	}
 
+	if err = stateTree.Insert(address, data); err != nil {
+		return errors.Wrap(err, "put pre coin proof")
+	}
+
 	return nil
 }
 
@@ -354,6 +387,7 @@ func (p *PebbleCoinStore) DeletePreCoinProof(
 	txn Transaction,
 	address []byte,
 	preCoinProof *protobufs.PreCoinProof,
+	stateTree *crypto.VectorCommitmentTree,
 ) error {
 	err := txn.Delete(proofKey(address))
 	if err != nil {
@@ -374,6 +408,10 @@ func (p *PebbleCoinStore) DeletePreCoinProof(
 		return errors.Wrap(err, "delete pre coin proof")
 	}
 
+	if err = stateTree.Delete(address); err != nil {
+		return errors.Wrap(err, "delete pre coin proof")
+	}
+
 	return nil
 }
 

node/tries/proof_leaf_test.go

@@ -89,7 +89,7 @@ func TestPackAndVerifyOutput(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			tree, payload, output, err := tries.PackOutputIntoPayloadAndProof(
+			tree, output, err := tries.PackOutputIntoPayloadAndProof(
 				outputs,
 				tc.modulo,
 				frame,
@@ -97,7 +97,6 @@ func TestPackAndVerifyOutput(t *testing.T) {
 			)
 			require.NoError(t, err)
 			require.NotNil(t, tree)
-			require.NotEmpty(t, payload)
 			require.NotEmpty(t, output)
 
 			var previousRoot []byte
@@ -116,23 +115,6 @@ func TestPackAndVerifyOutput(t *testing.T) {
 			require.Equal(t, uint32(tc.modulo), modulo, "Modulo mismatch")
 			require.Equal(t, tc.frameNum, frameNumber, "Frame number mismatch")
 
-			reconstructedPayload := []byte("mint")
-			reconstructedPayload = append(reconstructedPayload, treeRoot...)
-			reconstructedPayload = binary.BigEndian.AppendUint32(reconstructedPayload, modulo)
-			reconstructedPayload = binary.BigEndian.AppendUint64(reconstructedPayload, frameNumber)
-
-			if tc.withPrev {
-				for i := 3; i < len(output)-2; i++ {
-					reconstructedPayload = append(reconstructedPayload, output[i]...)
-				}
-				pathBytes := output[len(output)-2]
-				reconstructedPayload = append(reconstructedPayload, pathBytes...)
-				leafBytes := output[len(output)-1]
-				reconstructedPayload = append(reconstructedPayload, leafBytes...)
-			}
-
-			require.Equal(t, payload, reconstructedPayload, "Payload reconstruction mismatch")
-
 			if tc.withPrev {
 				t.Run("corrupted_proof", func(t *testing.T) {
 					corruptedOutput := make([][]byte, len(output))