ceremonyclient/node/execution/state/hypergraph/hypergraph_state.go
2025-12-15 16:45:31 -06:00

840 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package hypergraph
import (
"bytes"
"fmt"
"math/big"
"slices"
"sync"
"time"
"github.com/iden3/go-iden3-crypto/poseidon"
"github.com/pkg/errors"
hg "source.quilibrium.com/quilibrium/monorepo/hypergraph"
"source.quilibrium.com/quilibrium/monorepo/types/crypto"
"source.quilibrium.com/quilibrium/monorepo/types/execution/state"
"source.quilibrium.com/quilibrium/monorepo/types/hypergraph"
"source.quilibrium.com/quilibrium/monorepo/types/schema"
"source.quilibrium.com/quilibrium/monorepo/types/tries"
"source.quilibrium.com/quilibrium/monorepo/utils/p2p"
)
// Set to approximately 600 frames after deletion, we should reduce this as
// soon as we find a satisfactory distance.
const VERTEX_DATA_DELETION_INTERVAL = 10 * 60 * 1000
// Metadata entries for the hypergraph need to consistently occur at the same
// address see more details in the HypergraphState.Init method
var HYPERGRAPH_METADATA_ADDRESS = bytes.Repeat([]byte{0xff}, 32)
type HypergraphState struct {
mu sync.Mutex
hypergraph hypergraph.Hypergraph
changeset []state.StateChange
}
type VertexAddMaterializedState struct {
hypergraph *HypergraphState
appAddress [32]byte
dataAddress [32]byte
frameNumber uint64
prior *tries.VectorCommitmentTree
data *tries.VectorCommitmentTree
}
func (h *HypergraphState) NewVertexAddMaterializedState(
appAddress [32]byte,
dataAddress [32]byte,
frameNumber uint64,
prior *tries.VectorCommitmentTree,
data *tries.VectorCommitmentTree,
) *VertexAddMaterializedState {
if prior != nil {
prior.Commit(h.GetProver(), false)
}
if data != nil {
data.Commit(h.GetProver(), false)
}
return &VertexAddMaterializedState{
h,
appAddress,
dataAddress,
frameNumber,
prior,
data,
}
}
// Commit implements state.MaterializedState.
func (v *VertexAddMaterializedState) Commit(
txn tries.TreeBackingStoreTransaction,
) error {
prefix, err := v.hypergraph.hypergraph.GetCoveredPrefix()
if err != nil {
return errors.Wrap(err, "vertex add commit")
}
path := tries.GetFullPath(slices.Concat(v.appAddress[:], v.dataAddress[:]))
if !slices.Equal(path[:len(prefix)], prefix) {
return nil
}
if err := v.hypergraph.hypergraph.AddVertex(txn, hg.NewVertex(
v.appAddress,
v.dataAddress,
v.data.Commit(v.hypergraph.hypergraph.GetProver(), false),
v.data.GetSize(),
)); err != nil {
return errors.Wrap(err, "vertex add commit")
}
id := slices.Concat(v.appAddress[:], v.dataAddress[:])
err = v.hypergraph.hypergraph.SetVertexData(txn, [64]byte(id), v.data)
if err != nil {
return errors.Wrap(err, "vertex add commit")
}
shardKey := tries.ShardKey{
L1: [3]byte(p2p.GetBloomFilterIndices(id[:32], 256, 3)),
L2: [32]byte(append([]byte{}, id[:32]...)),
}
err = v.hypergraph.hypergraph.TrackChange(
txn,
id,
v.prior,
v.frameNumber,
string(hypergraph.AddsPhaseType),
string(hypergraph.VertexAtomType),
shardKey,
)
return errors.Wrap(err, "vertex add commit")
}
// DataValue implements state.MaterializedState.
func (v *VertexAddMaterializedState) DataValue() *tries.VectorCommitmentTree {
return v.data
}
func (v *VertexAddMaterializedState) GetVertex() hypergraph.Vertex {
return hg.NewVertex(
v.appAddress,
v.dataAddress,
v.data.Commit(v.hypergraph.hypergraph.GetProver(), false),
v.data.GetSize(),
)
}
type VertexRemoveMaterializedState struct {
hypergraph *HypergraphState
appAddress [32]byte
dataAddress [32]byte
deleteAt int64
frameNumber uint64
prior *tries.VectorCommitmentTree
commitment []byte
originalSize *big.Int
}
func (h *HypergraphState) NewVertexRemoveMaterializedState(
appAddress [32]byte,
dataAddress [32]byte,
deleteAt int64,
frameNumber uint64,
prior *tries.VectorCommitmentTree,
commitment []byte,
originalSize *big.Int,
) *VertexRemoveMaterializedState {
if prior != nil {
prior.Commit(h.GetProver(), false)
}
return &VertexRemoveMaterializedState{
h,
appAddress,
dataAddress,
deleteAt,
frameNumber,
prior,
commitment,
originalSize,
}
}
// Commit implements state.MaterializedState.
func (v *VertexRemoveMaterializedState) Commit(
txn tries.TreeBackingStoreTransaction,
) error {
prefix, err := v.hypergraph.hypergraph.GetCoveredPrefix()
if err != nil {
return errors.Wrap(err, "vertex add commit")
}
path := tries.GetFullPath(slices.Concat(v.appAddress[:], v.dataAddress[:]))
if !slices.Equal(path[:len(prefix)], prefix) {
return nil
}
id := slices.Concat(v.appAddress[:], v.dataAddress[:])
err = v.hypergraph.hypergraph.RemoveVertex(txn, hg.NewVertex(
v.appAddress,
v.dataAddress,
v.commitment,
v.originalSize,
))
if err != nil {
return errors.Wrap(err, "vertex remove commit")
}
shardKey := tries.ShardKey{
L1: [3]byte(p2p.GetBloomFilterIndices(id[:32], 256, 3)),
L2: [32]byte(append([]byte{}, id[:32]...)),
}
err = v.hypergraph.hypergraph.TrackChange(
txn,
id,
v.prior,
v.frameNumber,
string(hypergraph.RemovesPhaseType),
string(hypergraph.VertexAtomType),
shardKey,
)
return errors.Wrap(err, "vertex remove commit")
}
// DataValue implements state.MaterializedState.
func (
v *VertexRemoveMaterializedState,
) DataValue() *tries.VectorCommitmentTree {
return nil
}
type HyperedgeAddMaterializedState struct {
hypergraph *HypergraphState
frameNumber uint64
prior *tries.VectorCommitmentTree
value hypergraph.Hyperedge
}
func (h *HypergraphState) NewHyperedgeAddMaterializedState(
frameNumber uint64,
prior *tries.VectorCommitmentTree,
value hypergraph.Hyperedge,
) *HyperedgeAddMaterializedState {
if prior != nil {
prior.Commit(h.GetProver(), false)
}
if value != nil {
value.Commit(h.GetProver())
}
return &HyperedgeAddMaterializedState{
h,
frameNumber,
prior,
value,
}
}
// Commit implements state.MaterializedState.
func (h *HyperedgeAddMaterializedState) Commit(
txn tries.TreeBackingStoreTransaction,
) error {
prefix, err := h.hypergraph.hypergraph.GetCoveredPrefix()
if err != nil {
return errors.Wrap(err, "vertex add commit")
}
id := h.value.GetID()
path := tries.GetFullPath(id[:])
if !slices.Equal(path[:len(prefix)], prefix) {
return nil
}
err = h.hypergraph.hypergraph.AddHyperedge(txn, h.value)
if err != nil {
return errors.Wrap(err, "hyperedge add commit")
}
shardKey := tries.ShardKey{
L1: [3]byte(p2p.GetBloomFilterIndices(id[:32], 256, 3)),
L2: [32]byte(append([]byte{}, id[:32]...)),
}
err = h.hypergraph.hypergraph.TrackChange(
txn,
id[:],
h.prior,
h.frameNumber,
string(hypergraph.AddsPhaseType),
string(hypergraph.HyperedgeAtomType),
shardKey,
)
return errors.Wrap(err, "hyperedge add commit")
}
// DataValue implements state.MaterializedState.
func (
h *HyperedgeAddMaterializedState,
) DataValue() *tries.VectorCommitmentTree {
return h.value.GetExtrinsicTree()
}
type HyperedgeRemoveMaterializedState struct {
hypergraph *HypergraphState
frameNumber uint64
prior *tries.VectorCommitmentTree
value hypergraph.Hyperedge
}
func (h *HypergraphState) NewHyperedgeRemoveMaterializedState(
frameNumber uint64,
prior *tries.VectorCommitmentTree,
value hypergraph.Hyperedge,
) *HyperedgeRemoveMaterializedState {
if prior != nil {
prior.Commit(h.GetProver(), false)
}
if value != nil {
value.Commit(h.GetProver())
}
return &HyperedgeRemoveMaterializedState{
h,
frameNumber,
prior,
value,
}
}
// Commit implements state.MaterializedState.
func (h *HyperedgeRemoveMaterializedState) Commit(
txn tries.TreeBackingStoreTransaction,
) error {
prefix, err := h.hypergraph.hypergraph.GetCoveredPrefix()
if err != nil {
return errors.Wrap(err, "vertex add commit")
}
id := h.value.GetID()
path := tries.GetFullPath(id[:])
if !slices.Equal(path[:len(prefix)], prefix) {
return nil
}
err = h.hypergraph.hypergraph.RemoveHyperedge(txn, h.value)
if err != nil {
return errors.Wrap(err, "hyperedge remove commit")
}
shardKey := tries.ShardKey{
L1: [3]byte(p2p.GetBloomFilterIndices(id[:32], 256, 3)),
L2: [32]byte(append([]byte{}, id[:32]...)),
}
err = h.hypergraph.hypergraph.TrackChange(
txn,
id[:],
h.prior,
h.frameNumber,
string(hypergraph.RemovesPhaseType),
string(hypergraph.HyperedgeAtomType),
shardKey,
)
return errors.Wrap(err, "hyperedge remove commit")
}
// DataValue implements state.MaterializedState.
func (
h *HyperedgeRemoveMaterializedState,
) DataValue() *tries.VectorCommitmentTree {
return nil
}
var (
VertexAddsDiscriminator []byte
VertexRemovesDiscriminator []byte
HyperedgeAddsDiscriminator []byte
HyperedgeRemovesDiscriminator []byte
)
func init() {
vertexAddsDiscriminatorBI, _ := poseidon.HashBytes(
[]byte("vertex:adds"),
)
vertexRemovesDiscriminatorBI, _ := poseidon.HashBytes(
[]byte("vertex:removes"),
)
hyperedgeAddsDiscriminatorBI, _ := poseidon.HashBytes(
[]byte("hyperedge:adds"),
)
hyperedgeRemovesDiscriminatorBI, _ := poseidon.HashBytes(
[]byte("hyperedge:removes"),
)
VertexAddsDiscriminator = make([]byte, 32)
VertexRemovesDiscriminator = make([]byte, 32)
HyperedgeAddsDiscriminator = make([]byte, 32)
HyperedgeRemovesDiscriminator = make([]byte, 32)
vertexAddsDiscriminatorBI.FillBytes(VertexAddsDiscriminator)
vertexRemovesDiscriminatorBI.FillBytes(VertexRemovesDiscriminator)
hyperedgeAddsDiscriminatorBI.FillBytes(HyperedgeAddsDiscriminator)
hyperedgeRemovesDiscriminatorBI.FillBytes(HyperedgeRemovesDiscriminator)
}
func NewHypergraphState(hypergraph hypergraph.Hypergraph) *HypergraphState {
return &HypergraphState{
hypergraph: hypergraph,
changeset: []state.StateChange{},
}
}
func (h *HypergraphState) GetProver() crypto.InclusionProver {
return h.hypergraph.GetProver()
}
func (h *HypergraphState) sealMetadataStateAtIndex(
metadata, subData *tries.VectorCommitmentTree,
index byte,
name string,
) error {
if index > 63 {
return errors.Wrap(state.ErrInvalidData, "seal metadata state at index")
}
if subData == nil {
return errors.Wrap(
errors.Wrap(state.ErrInvalidData, name),
"seal metadata state at index",
)
}
subDataCommit := subData.Commit(
h.hypergraph.GetProver(),
false,
)
subDataBytes, err := tries.SerializeNonLazyTree(subData)
if err != nil {
return errors.Wrap(errors.Wrap(err, name), "seal metadata state at index")
}
err = metadata.Insert(
[]byte{index << 2}, // We move it up two bits to be in the first nibble
subDataBytes,
subDataCommit,
subData.GetSize(),
)
if err != nil {
return errors.Wrap(errors.Wrap(err, name), "seal metadata state at index")
}
return nil
}
func unsealMetadataStateAtIndex(
metadata *tries.VectorCommitmentTree,
index byte,
name string,
) (*tries.VectorCommitmentTree, error) {
if index > 63 {
return nil, errors.Wrap(
state.ErrInvalidData,
"unseal metadata state at index",
)
}
// We move it up two bits to be in the first nibble
leaf, err := metadata.Get([]byte{index << 2})
if err != nil {
return nil, errors.Wrap(err, "unseal metadata state at index")
}
subTree, err := tries.DeserializeNonLazyTree(leaf)
if err != nil {
return nil, errors.Wrap(
errors.Wrap(err, name),
"unseal metadata state at index",
)
}
return subTree, nil
}
func UnpackConsensusMetadata(
tree *tries.VectorCommitmentTree,
) (*tries.VectorCommitmentTree, error) {
consensusMetadata, err := unsealMetadataStateAtIndex(
tree,
0,
"consensus metadata",
)
if err != nil {
return nil, errors.Wrap(err, "unpack consensus metadata")
}
return consensusMetadata, nil
}
func UnpackSumcheckInfo(
tree *tries.VectorCommitmentTree,
) (*tries.VectorCommitmentTree, error) {
sumcheckInfo, err := unsealMetadataStateAtIndex(
tree,
1,
"sumcheck info",
)
if err != nil {
return nil, errors.Wrap(err, "unpack sumcheck info")
}
return sumcheckInfo, nil
}
func UnpackRdfHypergraphSchema(
tree *tries.VectorCommitmentTree,
) (string, error) {
leaf, err := tree.Get([]byte{2 << 2})
if err != nil {
return "", errors.Wrap(err, "unpack rdf indices")
}
if len(leaf) == 0 {
return "", nil
}
valid, err := (&schema.TurtleRDFParser{}).Validate(string(leaf))
if err != nil {
return "", errors.Wrap(err, "unpack rdf indices")
}
if !valid {
return "", errors.Wrap(errors.New("invalid schema"), "unpack rdf indices")
}
return string(leaf), nil
}
// Init implements state.State for hypergraphs, as an app-level shard.
// Hypergraph initialization only creates a vertex addition, as an empty VCT,
// at an address impossible to reach via regular address derivation, so as to
// ensure whatever metadata it holds does not need to adhere to the same
// encoding expectation as regular data stored in the hypergraph, so whatever
// necessary public state information needs to be held will not have risk of
// triggering confusion-based collision (as well as ensuring that metadata
// remains at a maximum branch distance of 1 from the domain root). VCT metadata
// encoding MUST include shard-level consensus metadata at index 0 (intrinsic-
// controlled and interpreted for extensibility purposes), MUST include relevant
// sumcheck info at index 1, MAY include RDF->HG index mapping at index 2,
// indices 3-15 are preserved for future use in general hypergraph metadata,
// indices 16-62 are left to intrinsic-specific use cases, 63 is the type
// designator.
func (h *HypergraphState) Init(
domain []byte,
consensusMetadata *tries.VectorCommitmentTree,
sumcheckInfo *tries.VectorCommitmentTree,
rdfSchema string,
additionalData []*tries.VectorCommitmentTree,
intrinsicType []byte,
) error {
if len(domain) != 32 {
return errors.Wrap(state.ErrInvalidDomain, "init")
}
if len(additionalData) > 62 {
return errors.Wrap(state.ErrInvalidData, "init")
}
publicStateInformation := &tries.VectorCommitmentTree{}
if err := h.sealMetadataStateAtIndex(
publicStateInformation,
consensusMetadata,
0,
"consensus metadata",
); err != nil {
return errors.Wrap(err, "init")
}
if err := h.sealMetadataStateAtIndex(
publicStateInformation,
sumcheckInfo,
1,
"sumcheck info",
); err != nil {
return errors.Wrap(err, "init")
}
if err := publicStateInformation.Insert(
[]byte{2 << 2},
[]byte(rdfSchema),
nil,
big.NewInt(int64(len(rdfSchema))),
); err != nil {
return errors.Wrap(err, "init")
}
if len(additionalData) > 59 {
return errors.Wrap(errors.New("reserved metadata index"), "init")
}
if len(additionalData) > 0 {
for i, add := range additionalData {
index := i + 3
if index < 16 {
if add != nil {
return errors.Wrap(errors.New("reserved metadata index"), "init")
} else {
continue
}
}
if err := h.sealMetadataStateAtIndex(
publicStateInformation,
add,
byte(index),
fmt.Sprintf("parent intrinsic at index %d", i),
); err != nil {
return errors.Wrap(err, "init")
}
}
}
if err := publicStateInformation.Insert(
bytes.Repeat([]byte{0xff}, 32),
intrinsicType,
nil,
big.NewInt(int64(len(intrinsicType))),
); err != nil {
return errors.Wrap(err, "init")
}
initializedDomain := make([]byte, 32)
copy(initializedDomain, domain)
h.mu.Lock()
h.changeset = append(h.changeset, state.StateChange{
Domain: initializedDomain,
Address: HYPERGRAPH_METADATA_ADDRESS,
Discriminator: VertexAddsDiscriminator,
StateChange: state.InitializeStateChangeEvent,
Value: &VertexAddMaterializedState{
hypergraph: h,
appAddress: [32]byte(initializedDomain),
dataAddress: [32]byte(HYPERGRAPH_METADATA_ADDRESS),
data: publicStateInformation,
},
})
h.mu.Unlock()
return nil
}
// Changeset implements state.State, as a list of hypergraph state changes. The
// returned slice is the underlying reference upstream intrinsics MUST take
// caution to not mutate it.
func (h *HypergraphState) Changeset() []state.StateChange {
return h.changeset
}
// Delete implements state.State as a deletion event on a hypergraph.
func (h *HypergraphState) Delete(
domain []byte,
address []byte,
discriminator []byte,
frameNumber uint64,
) error {
id := [64]byte{}
copy(id[:32], domain)
copy(id[32:], address)
var value state.MaterializedState
if bytes.Equal(discriminator, VertexRemovesDiscriminator) {
vertex, err := h.hypergraph.GetVertex(id)
if err != nil {
return errors.Wrap(err, "delete")
}
data, err := h.hypergraph.GetVertexData(id)
if err != nil {
return errors.Wrap(err, "delete")
}
value = h.NewVertexRemoveMaterializedState(
[32]byte(domain),
[32]byte(address),
time.Now().UnixMilli()+VERTEX_DATA_DELETION_INTERVAL,
frameNumber,
data,
vertex.Commit(nil),
vertex.GetSize(),
)
} else if bytes.Equal(discriminator, HyperedgeRemovesDiscriminator) {
hyperedge, err := h.hypergraph.GetHyperedge(id)
if err != nil {
return errors.Wrap(err, "delete")
}
value = h.NewHyperedgeRemoveMaterializedState(
frameNumber,
hyperedge.GetExtrinsicTree(),
hyperedge,
)
} else {
return errors.Wrap(state.ErrInvalidDiscriminator, "delete")
}
h.mu.Lock()
h.changeset = append(h.changeset, state.StateChange{
Domain: domain, // buildutils:allow-slice-alias slice is static
Address: address, // buildutils:allow-slice-alias slice is static
Discriminator: discriminator, // buildutils:allow-slice-alias slice is static
StateChange: state.DeleteStateChangeEvent,
Value: value,
})
h.mu.Unlock()
return nil
}
// Get implements state.State as a simple fetcher on the hypergraph.
// Discriminator is used to distinguish between set types (vertex vs hyperedge).
func (h *HypergraphState) Get(
domain []byte,
address []byte,
discriminator []byte,
) (interface{}, error) {
h.mu.Lock()
for _, c := range slices.Backward(h.changeset) {
if bytes.Equal(c.Address, address) && bytes.Equal(c.Domain, domain) &&
bytes.Equal(c.Discriminator, discriminator) {
h.mu.Unlock()
return c.Value.DataValue(), nil
}
}
h.mu.Unlock()
id := [64]byte{}
copy(id[:32], domain)
copy(id[32:], address)
if bytes.Equal(discriminator, VertexAddsDiscriminator) {
data, err := h.hypergraph.GetVertexData(id)
if err != nil {
return nil, errors.Wrap(err, "get")
}
return data, errors.Wrap(err, "get")
} else if bytes.Equal(discriminator, HyperedgeAddsDiscriminator) {
he, err := h.hypergraph.GetHyperedge(id)
if err != nil {
return nil, errors.Wrap(err, "get")
}
return he, nil
}
return nil, errors.Wrap(state.ErrInvalidDiscriminator, "get")
}
// Set implements state.State as a simple setter on the hypergraph.
// Discriminator is used to distinguish between set types (vertex vs hyperedge).
func (h *HypergraphState) Set(
domain []byte,
address []byte,
discriminator []byte,
frameNumber uint64,
value state.MaterializedState,
) error {
id := [64]byte{}
copy(id[:32], domain)
copy(id[32:], address)
previousValue := false
if bytes.Equal(discriminator, VertexAddsDiscriminator) {
if _, ok := value.(*VertexAddMaterializedState); !ok {
return errors.Wrap(state.ErrInvalidDiscriminator, "set")
}
vertex, err := h.hypergraph.GetVertex(id)
if err != nil && errors.Is(err, hypergraph.ErrRemoved) {
return errors.Wrap(err, "set")
}
if vertex != nil {
previousValue = true
}
} else if bytes.Equal(discriminator, HyperedgeAddsDiscriminator) {
if _, ok := value.(*HyperedgeAddMaterializedState); !ok {
return errors.Wrap(state.ErrInvalidDiscriminator, "set")
}
hyperedge, err := h.hypergraph.GetHyperedge(id)
if err != nil && errors.Is(err, hypergraph.ErrRemoved) {
return errors.Wrap(err, "set")
}
if hyperedge != nil {
previousValue = true
}
} else {
return errors.Wrap(state.ErrInvalidDiscriminator, "set")
}
stateChange := state.CreateStateChangeEvent
if previousValue {
stateChange = state.UpdateStateChangeEvent
}
h.mu.Lock()
h.changeset = append(h.changeset, state.StateChange{
Domain: domain, // buildutils:allow-slice-alias slice is static
Address: address, // buildutils:allow-slice-alias slice is static
Discriminator: discriminator, // buildutils:allow-slice-alias slice is static
StateChange: stateChange,
Value: value,
})
h.mu.Unlock()
return nil
}
// Commit implements state.State, committing the (db-level) transaction set.
func (h *HypergraphState) Commit() error {
txn, err := h.hypergraph.NewTransaction(false)
if err != nil {
return errors.Wrap(err, "commit")
}
h.mu.Lock()
defer h.mu.Unlock()
for _, change := range h.changeset {
if err := change.Value.Commit(txn); err != nil {
if err := txn.Abort(); err != nil {
return errors.Wrap(err, "commit")
}
return errors.Wrap(err, "commit")
}
}
if err := txn.Commit(); err != nil {
return errors.Wrap(err, "commit")
}
return nil
}
// Abort implements state.State, aborting the (db-level) transaction set.
func (h *HypergraphState) Abort() error {
h.mu.Lock()
h.changeset = []state.StateChange{}
h.mu.Unlock()
return nil
}
var _ state.State = (*HypergraphState)(nil)
var _ state.MaterializedState = (*VertexAddMaterializedState)(nil)
var _ state.MaterializedState = (*VertexRemoveMaterializedState)(nil)
var _ state.MaterializedState = (*HyperedgeAddMaterializedState)(nil)
var _ state.MaterializedState = (*HyperedgeRemoveMaterializedState)(nil)