From a0107caabc3b6275239bf89873e7f28d4228bfff Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Mon, 27 Jan 2025 04:23:56 -0600 Subject: [PATCH 01/13] tree rebuild at fork --- Cargo.lock | 72 ++- Cargo.toml | 1 + crates/channel-wasm/Cargo.toml | 7 + crates/channel-wasm/src/lib.rs | 480 ++++++++++++++++++ crates/channel/src/lib.rs | 126 ++++- crates/channel/src/protocols/feldman.rs | 92 +++- crates/channel/src/protocols/tripleratchet.rs | 64 ++- crates/ed448-rust/.editorconfig | 10 + crates/ed448-rust/.github/workflows/audit.yml | 28 + crates/ed448-rust/.github/workflows/ci.yml | 171 +++++++ crates/ed448-rust/.gitignore | 4 + crates/ed448-rust/CHANGELOG.md | 14 + crates/ed448-rust/Cargo.toml | 55 ++ crates/ed448-rust/LICENSE_APACHE2.txt | 175 +++++++ crates/ed448-rust/LICENSE_MIT.txt | 21 + crates/ed448-rust/README.md | 68 +++ crates/ed448-rust/src/error.rs | 46 ++ crates/ed448-rust/src/lib.rs | 291 +++++++++++ crates/ed448-rust/src/point.rs | 433 ++++++++++++++++ crates/ed448-rust/src/private_key.rs | 255 ++++++++++ crates/ed448-rust/src/public_key.rs | 293 +++++++++++ crates/ed448-rust/tests/rfc8032.rs | 215 ++++++++ node/config/version.go | 4 +- .../data/data_clock_consensus_engine.go | 2 - node/consensus/data/peer_messaging.go | 22 + .../data/pre_midnight_proof_worker.go | 300 ----------- node/crypto/proof_tree.go | 12 +- node/crypto/proof_tree_test.go | 111 +++- .../token/application/token_handle_mint.go | 8 +- .../token/token_execution_engine.go | 39 +- node/hypergraph/application/hypergraph.go | 38 +- .../hypergraph_convergence_test.go | 7 +- .../hypergraph/application/hypergraph_test.go | 34 +- node/test.sh | 2 +- 34 files changed, 3103 insertions(+), 397 deletions(-) create mode 100644 crates/ed448-rust/.editorconfig create mode 100644 crates/ed448-rust/.github/workflows/audit.yml create mode 100644 crates/ed448-rust/.github/workflows/ci.yml create mode 100644 crates/ed448-rust/.gitignore create mode 100644 crates/ed448-rust/CHANGELOG.md create mode 100644 crates/ed448-rust/Cargo.toml create mode 100644 crates/ed448-rust/LICENSE_APACHE2.txt create mode 100644 crates/ed448-rust/LICENSE_MIT.txt create mode 100644 crates/ed448-rust/README.md create mode 100644 crates/ed448-rust/src/error.rs create mode 100644 crates/ed448-rust/src/lib.rs create mode 100644 crates/ed448-rust/src/point.rs create mode 100644 crates/ed448-rust/src/private_key.rs create mode 100644 crates/ed448-rust/src/public_key.rs create mode 100644 crates/ed448-rust/tests/rfc8032.rs delete mode 100644 node/consensus/data/pre_midnight_proof_worker.go diff --git a/Cargo.lock b/Cargo.lock index 6790c20..432e783 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -231,12 +237,22 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ - "block-padding", + "block-padding 0.1.5", "byte-tools", "byteorder", "generic-array 0.12.4", ] +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "block-padding 0.2.1", + "generic-array 0.14.7", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -255,6 +271,12 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "bls48581" version = "0.1.0" @@ -348,7 +370,7 @@ name = "channel" version = "0.1.0" dependencies = [ "aes-gcm", - "base64", + "base64 0.22.1", "criterion 0.4.0", "ed448-goldilocks-plus", "hex 0.4.3", @@ -367,10 +389,17 @@ dependencies = [ name = "channelwasm" version = "0.1.0" dependencies = [ + "aes-gcm", + "base64 0.22.1", "channel", + "ed448-goldilocks-plus", + "ed448-rust", "getrandom", + "hkdf", + "rand", "serde", "serde_json", + "sha2 0.10.8", "wasm-bindgen", ] @@ -674,6 +703,15 @@ dependencies = [ "generic-array 0.12.4", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "digest" version = "0.10.7" @@ -695,11 +733,27 @@ dependencies = [ "hex 0.4.3", "rand_core", "serde", - "sha3", + "sha3 0.10.8", "subtle", "zeroize", ] +[[package]] +name = "ed448-rust" +version = "0.1.2" +dependencies = [ + "base64 0.13.1", + "hex 0.4.3", + "lazy_static", + "num-bigint", + "num-integer", + "num-traits", + "opaque-debug 0.3.1", + "rand_core", + "sha3 0.9.1", + "subtle", +] + [[package]] name = "either" version = "1.12.0" @@ -1474,6 +1528,18 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug 0.3.1", +] + [[package]] name = "sha3" version = "0.10.8" diff --git a/Cargo.toml b/Cargo.toml index a4c8196..13ef076 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "crates/channel-wasm", "crates/classgroup", "crates/bls48581", + "crates/ed448-rust", "crates/rpm", ] diff --git a/crates/channel-wasm/Cargo.toml b/crates/channel-wasm/Cargo.toml index 14fed50..957e94a 100644 --- a/crates/channel-wasm/Cargo.toml +++ b/crates/channel-wasm/Cargo.toml @@ -13,3 +13,10 @@ channel = { path = "../channel", version = "^0.1.0" } getrandom = { version = "0.2", features = ["js"] } serde = "1.0.208" serde_json = "1.0.117" +base64 = "0.22.1" +ed448-goldilocks-plus = "0.11.2" +ed448-rust = { path = "../ed448-rust", version = "0.1.2" } +rand = "0.8.5" +sha2 = "0.10.8" +hkdf = "0.12.4" +aes-gcm = "0.10.3" \ No newline at end of file diff --git a/crates/channel-wasm/src/lib.rs b/crates/channel-wasm/src/lib.rs index b705f19..07302a1 100644 --- a/crates/channel-wasm/src/lib.rs +++ b/crates/channel-wasm/src/lib.rs @@ -1,6 +1,14 @@ +use aes_gcm::{Aes256Gcm, Nonce}; +use aes_gcm::aead::{Aead, Payload}; +use ed448_rust::Ed448Error; +use hkdf::Hkdf; +use rand::{rngs::OsRng, RngCore}; +use sha2::Sha512; use wasm_bindgen::prelude::*; +use base64::prelude::*; use channel::*; use serde::{Deserialize, Serialize}; +use ed448_goldilocks_plus::{elliptic_curve::Group, CompressedEdwardsY, EdwardsPoint, Scalar}; #[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct NewDoubleRatchetParameters { @@ -22,6 +30,433 @@ pub struct NewTripleRatchetParameters { pub async_dkg_ratchet: bool } +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct EncryptionKeyPair { + pub public_key: Vec, + pub private_key: Vec, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct SigningKeyPair { + pub public_key: Vec, + pub private_key: Vec, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct SenderX3DH { + pub sending_identity_private_key: Vec, + pub sending_ephemeral_private_key: Vec, + pub receiving_identity_key: Vec, + pub receiving_signed_pre_key: Vec, + pub session_key_length: usize, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct ReceiverX3DH { + pub sending_identity_private_key: Vec, + pub sending_signed_private_key: Vec, + pub receiving_identity_key: Vec, + pub receiving_ephemeral_key: Vec, + pub session_key_length: usize, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)] +pub struct MessageCiphertext { + pub ciphertext: String, + pub initialization_vector: String, + pub associated_data: Option, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct SealedInboxMessageDecryptRequest { + pub inbox_private_key: Vec, + pub ephemeral_public_key: Vec, + pub ciphertext: MessageCiphertext, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct SealedInboxMessageEncryptRequest { + pub inbox_public_key: Vec, + pub ephemeral_private_key: Vec, + pub plaintext: Vec, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct ResizeRequest { + pub ratchet_state: String, + pub other: String, + pub id: usize, + pub total: usize, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct TripleRatchetStateAndPoint { + pub ratchet_state: String, + pub point: String, + pub index: usize, +} + + +fn encrypt(plaintext: &[u8], key: &[u8]) + -> Result> { + use aes_gcm::KeyInit; + let mut iv = [0u8; 12]; + OsRng.fill_bytes(&mut iv); + + let cipher = Aes256Gcm::new_from_slice(key).unwrap(); + let nonce = Nonce::from_slice(&iv); + + let mut aad = [0u8; 32]; + OsRng.fill_bytes(&mut aad); + + let ciphertext = cipher.encrypt(nonce, Payload{ + msg: plaintext, + aad: &aad, + }).map_err(|e| format!("Encryption failed: {}", e))?; + + Ok(MessageCiphertext { + ciphertext: BASE64_STANDARD.encode(ciphertext), + initialization_vector: BASE64_STANDARD.encode(iv.to_vec()), + associated_data: Some(BASE64_STANDARD.encode(aad.to_vec())), + }) +} + +fn decrypt(ciphertext: &MessageCiphertext, key: &[u8]) + -> Result, Box> { + use aes_gcm::KeyInit; + if key.len() != 32 { + return Err(format!("Invalid key length").into()); + } + let cipher = Aes256Gcm::new_from_slice(key).unwrap(); + let maybe_iv = BASE64_STANDARD.decode(&ciphertext.initialization_vector); + if maybe_iv.is_err() { + return Err(format!("Decryption failed: {}", maybe_iv.unwrap_err()).into()); + } + + let iv = maybe_iv.unwrap(); + let nonce = Nonce::from_slice(&iv); + + let ad = &ciphertext.associated_data; + let mut associated_data = Vec::::new(); + if ad.is_some() { + let aad = BASE64_STANDARD.decode(&ad.clone().unwrap()); + if aad.is_err() { + return Err(format!("Decryption failed: {}", aad.unwrap_err()).into()); + } + + associated_data = aad.unwrap(); + } + + let ciphertext = BASE64_STANDARD.decode(&ciphertext.ciphertext); + if ciphertext.is_err() { + return Err(format!("Decryption failed: {}", ciphertext.unwrap_err()).into()); + } + + cipher.decrypt(nonce, Payload{ + msg: &ciphertext.unwrap(), + aad: &associated_data, + }).map_err(|e| format!("Decryption failed: {}", e).into()) +} + +#[wasm_bindgen] +pub fn js_decrypt_inbox_message(input: &str) -> String { + let json: Result = serde_json::from_str(input); + match json { + Ok(params) => { + let ephemeral_key = params.ephemeral_public_key; + if ephemeral_key.len() != 57 { + return "invalid ephemeral key length".to_string(); + } + + let inbox_key = params.inbox_private_key; + if inbox_key.len() != 56 { + return "invalid inbox key length".to_string(); + } + + let ephemeral_key_bytes: [u8; 57] = ephemeral_key.try_into().unwrap(); + let inbox_key_bytes: [u8; 56] = inbox_key.try_into().unwrap(); + let priv_key = Scalar::from_bytes(&inbox_key_bytes); + let maybe_eph_key = CompressedEdwardsY(ephemeral_key_bytes).decompress(); + + if maybe_eph_key.is_none().into() { + return "invalid ephemeral key".to_string(); + } + + let dh_output = priv_key * maybe_eph_key.unwrap(); + // TODO: To support authorized inbox sending, change None to Some when an authorization_key is present. + let hkdf = Hkdf::::new(None, &dh_output.compress().to_bytes()); + let mut derived = [0u8; 32]; + let err = hkdf.expand(b"quilibrium-sealed-sender", &mut derived); + if err.is_err() { + return "invalid length".into(); + } + + let result = decrypt(¶ms.ciphertext, &derived); + if result.is_err() { + return result.unwrap_err().to_string(); + } + + match serde_json::to_string(&result.unwrap()) { + Ok(result) => result, + Err(e) => e.to_string(), + } + } + Err(e) => { + return e.to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_encrypt_inbox_message(input: &str) -> String { + let json: Result = serde_json::from_str(input); + match json { + Ok(params) => { + let key = params.ephemeral_private_key; + if key.len() != 56 { + return "invalid ephemeral key length".to_string(); + } + + let inbox_key = params.inbox_public_key; + if inbox_key.len() != 57 { + return "invalid inbox key length".to_string(); + } + + let key_bytes: [u8; 56] = key.try_into().unwrap(); + let inbox_key_bytes: [u8; 57] = inbox_key.try_into().unwrap(); + let priv_key = Scalar::from_bytes(&key_bytes); + let maybe_pub_key = CompressedEdwardsY(inbox_key_bytes).decompress(); + + if maybe_pub_key.is_none().into() { + return "invalid inbox key".to_string(); + } + + let dh_output = priv_key * maybe_pub_key.unwrap(); + // TODO: To support authorized inbox sending, change None to Some when an authorization_key is present. + let hkdf = Hkdf::::new(None, &dh_output.compress().to_bytes()); + let mut derived = [0u8; 32]; + let err = hkdf.expand(b"quilibrium-sealed-sender", &mut derived); + if err.is_err() { + return "invalid length".into(); + } + + let result = encrypt(¶ms.plaintext, &derived); + if result.is_err() { + return result.unwrap_err().to_string(); + } + + match serde_json::to_string(&result.unwrap()) { + Ok(result) => result, + Err(e) => e.to_string(), + } + } + Err(e) => { + return e.to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_sender_x3dh(input: &str) -> String { + let json: Result = serde_json::from_str(input); + match json { + Ok(params) => { + return sender_x3dh(¶ms.sending_identity_private_key, ¶ms.sending_ephemeral_private_key, ¶ms.receiving_identity_key, ¶ms.receiving_signed_pre_key, params.session_key_length); + } + Err(e) => { + return e.to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_receiver_x3dh(input: &str) -> String { + let json: Result = serde_json::from_str(input); + match json { + Ok(params) => { + return receiver_x3dh(¶ms.sending_identity_private_key, ¶ms.sending_signed_private_key, ¶ms.receiving_identity_key, ¶ms.receiving_ephemeral_key, params.session_key_length); + } + Err(e) => { + return e.to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_generate_x448() -> String { + let priv_key = Scalar::random(&mut rand::thread_rng()); + let pub_key = EdwardsPoint::generator() * priv_key; + let output = serde_json::to_string(&EncryptionKeyPair{ + public_key: pub_key.compress().to_bytes().to_vec(), + private_key: priv_key.to_bytes().to_vec(), + }); + + match output { + Ok(result) => { + result + } + Err(e) => { + e.to_string() + } + } +} + +#[wasm_bindgen] +pub fn js_generate_ed448() -> String { + let priv_key = ed448_rust::PrivateKey::new(&mut rand::thread_rng()); + let pub_key = ed448_rust::PublicKey::from(&priv_key); + let output = serde_json::to_string(&EncryptionKeyPair{ + public_key: pub_key.as_byte().to_vec(), + private_key: priv_key.as_bytes().to_vec(), + }); + + match output { + Ok(result) => { + result + } + Err(e) => { + e.to_string() + } + } +} + +#[wasm_bindgen] +pub fn js_get_pubkey_ed448(key: &str) -> String { + let maybe_key = BASE64_STANDARD.decode(key); + if maybe_key.is_err() { + return maybe_key.unwrap_err().to_string(); + } + + let key = maybe_key.unwrap(); + if key.len() != 57 { + return "invalid key length".to_string(); + } + + let key_bytes: [u8; 57] = key.try_into().unwrap(); + let priv_key = ed448_rust::PrivateKey::from(key_bytes); + let pub_key = ed448_rust::PublicKey::from(&priv_key); + + return format!("\"{}\"", BASE64_STANDARD.encode(pub_key.as_byte())); +} + +#[wasm_bindgen] +pub fn js_get_pubkey_x448(key: &str) -> String { + let maybe_key = BASE64_STANDARD.decode(key); + if maybe_key.is_err() { + return maybe_key.unwrap_err().to_string(); + } + + let key = maybe_key.unwrap(); + if key.len() != 56 { + return "invalid key length".to_string(); + } + + let mut priv_key_bytes = [0u8; 56]; + priv_key_bytes.copy_from_slice(&key); + + let priv_key = Scalar::from_bytes(&priv_key_bytes); + let pub_key = EdwardsPoint::generator() * priv_key; + + return format!("\"{}\"", BASE64_STANDARD.encode(pub_key.compress().to_bytes().to_vec())); +} + +#[wasm_bindgen] +pub fn js_sign_ed448(key: &str, message: &str) -> String { + let maybe_key = BASE64_STANDARD.decode(key); + if maybe_key.is_err() { + return maybe_key.unwrap_err().to_string(); + } + + let maybe_message = BASE64_STANDARD.decode(message); + if maybe_message.is_err() { + return maybe_message.unwrap_err().to_string(); + } + + let key = maybe_key.unwrap(); + if key.len() != 57 { + return "invalid key length".to_string(); + } + + let key_bytes: [u8; 57] = key.try_into().unwrap(); + let priv_key = ed448_rust::PrivateKey::from(key_bytes); + let signature = priv_key.sign(&maybe_message.unwrap(), None); + + match signature { + Ok(output) => { + return format!("\"{}\"", BASE64_STANDARD.encode(output)); + } + Err(Ed448Error::WrongKeyLength) => { + return "invalid key length".to_string(); + } + Err(Ed448Error::WrongPublicKeyLength) => { + return "invalid public key length".to_string(); + } + Err(Ed448Error::WrongSignatureLength) => { + return "invalid signature length".to_string(); + } + Err(Ed448Error::InvalidPoint) => { + return "invalid point".to_string(); + } + Err(Ed448Error::InvalidSignature) => { + return "invalid signature".to_string(); + } + Err(Ed448Error::ContextTooLong) => { + return "context too long".to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_verify_ed448(public_key: &str, message: &str, signature: &str) -> String { + let maybe_key = BASE64_STANDARD.decode(public_key); + if maybe_key.is_err() { + return maybe_key.unwrap_err().to_string(); + } + + let maybe_message = BASE64_STANDARD.decode(message); + if maybe_message.is_err() { + return maybe_message.unwrap_err().to_string(); + } + + let maybe_signature = BASE64_STANDARD.decode(signature); + if maybe_signature.is_err() { + return maybe_signature.unwrap_err().to_string(); + } + + let key = maybe_key.unwrap(); + if key.len() != 57 { + return "invalid key length".to_string(); + } + + let pub_bytes: [u8; 57] = key.try_into().unwrap(); + let pub_key = ed448_rust::PublicKey::from(pub_bytes); + let signature = pub_key.verify(&maybe_message.unwrap(), &maybe_signature.unwrap(), None); + + match signature { + Ok(()) => { + return "true".to_string(); + } + Err(Ed448Error::WrongKeyLength) => { + return "invalid key length".to_string(); + } + Err(Ed448Error::WrongPublicKeyLength) => { + return "invalid public key length".to_string(); + } + Err(Ed448Error::WrongSignatureLength) => { + return "invalid signature length".to_string(); + } + Err(Ed448Error::InvalidPoint) => { + return "invalid point".to_string(); + } + Err(Ed448Error::InvalidSignature) => { + return "invalid signature".to_string(); + } + Err(Ed448Error::ContextTooLong) => { + return "context too long".to_string(); + } + } +} + #[wasm_bindgen] pub fn js_new_double_ratchet(params: &str) -> String { let json: Result = serde_json::from_str(params); @@ -152,3 +587,48 @@ pub fn js_triple_ratchet_decrypt(params: &str) -> String { } } +#[wasm_bindgen] +pub fn js_triple_ratchet_resize(params: &str) -> String { + let json: Result = serde_json::from_str(params); + match json { + Ok(request) => { + return serde_json::to_string(&triple_ratchet_resize(request.ratchet_state, request.other, request.id, request.total)).unwrap_or_else(|e| e.to_string()); + } + Err(e) => { + return e.to_string(); + } + } +} + +#[wasm_bindgen] +pub fn js_verify_point(params: &str) -> String { + let json: Result = serde_json::from_str(params); + match json { + Ok(request) => { + let verify = triple_ratchet_verify_point(request.ratchet_state, request.point, request.index); + return match verify { + Ok(result) => serde_json::to_string(&result).unwrap_or_else(|e| e.to_string()), + Err(e) => serde_json::to_string(&e.to_string()).unwrap_or_else(|e| e.to_string()) + } + } + Err(e) => { + return e.to_string(); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use ed448_goldilocks_plus::{Scalar, elliptic_curve::Group, EdwardsPoint}; + + #[test] + fn test_verify() { + let priv_key = ed448_rust::PrivateKey::new(&mut rand::thread_rng()); + let pub_key = ed448_rust::PublicKey::from(&priv_key); + let sig = js_sign_ed448(&BASE64_STANDARD.encode(priv_key.as_bytes()).to_string(), "AQAB"); + assert_eq!(js_verify_ed448(&BASE64_STANDARD.encode(pub_key.as_byte()).to_string(), "AQAB", &serde_json::from_str::(&sig.to_string()).unwrap()), "true") + } +} \ No newline at end of file diff --git a/crates/channel/src/lib.rs b/crates/channel/src/lib.rs index 682eb5c..22d20ca 100644 --- a/crates/channel/src/lib.rs +++ b/crates/channel/src/lib.rs @@ -1,9 +1,9 @@ use base64::prelude::*; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::{collections::HashMap, error::Error, io::Read}; -use ed448_goldilocks_plus::{elliptic_curve::group::GroupEncoding, EdwardsPoint, Scalar}; -use protocols::{doubleratchet::{DoubleRatchetParticipant, P2PChannelEnvelope}, tripleratchet::{PeerInfo, TripleRatchetParticipant}}; +use ed448_goldilocks_plus::{elliptic_curve::group::GroupEncoding, CompressedEdwardsY, EdwardsPoint, Scalar}; +use protocols::{doubleratchet::{DoubleRatchetParticipant, P2PChannelEnvelope}, tripleratchet::{PeerInfo, TripleRatchetParticipant}, x3dh}; pub(crate) mod protocols; @@ -39,6 +39,86 @@ pub struct TripleRatchetStateAndMessage { pub message: Vec, } +pub fn sender_x3dh(sending_identity_private_key: &Vec, sending_ephemeral_private_key: &Vec, receiving_identity_key: &Vec, receiving_signed_pre_key: &Vec, session_key_length: usize) -> String { + if sending_identity_private_key.len() != 56 { + return "invalid sending identity private key length".to_string(); + } + + if sending_ephemeral_private_key.len() != 56 { + return "invalid sending ephemeral private key length".to_string(); + } + + if receiving_identity_key.len() != 57 { + return "invalid receiving identity public key length".to_string(); + } + + if receiving_signed_pre_key.len() != 57 { + return "invalid receiving signed public key length".to_string(); + } + + let sidk = Scalar::from_bytes(sending_identity_private_key.as_slice().try_into().unwrap()); + let sepk = Scalar::from_bytes(sending_ephemeral_private_key.as_slice().try_into().unwrap()); + let ridk = CompressedEdwardsY(receiving_identity_key.as_slice().try_into().unwrap()).decompress(); + let rspk = CompressedEdwardsY(receiving_signed_pre_key.as_slice().try_into().unwrap()).decompress(); + + if ridk.is_none().into() { + return "invalid receiving identity public key".to_string(); + } + + if rspk.is_none().into() { + return "invalid receiving signed public key".to_string(); + } + + match x3dh::sender_x3dh(&sidk, &sepk, &ridk.unwrap(), &rspk.unwrap(), session_key_length) { + Some(result) => { + return format!("\"{}\"", BASE64_STANDARD.encode(result)); + } + None => { + return "could not perform key agreement".to_string(); + } + } +} + +pub fn receiver_x3dh(sending_identity_private_key: &Vec, sending_signed_private_key: &Vec, receiving_identity_key: &Vec, receiving_ephemeral_key: &Vec, session_key_length: usize) -> String { + if sending_identity_private_key.len() != 56 { + return "invalid sending identity private key length".to_string(); + } + + if sending_signed_private_key.len() != 56 { + return "invalid sending signed private key length".to_string(); + } + + if receiving_identity_key.len() != 57 { + return "invalid receiving identity public key length".to_string(); + } + + if receiving_ephemeral_key.len() != 57 { + return "invalid receiving ephemeral public key length".to_string(); + } + + let sidk = Scalar::from_bytes(sending_identity_private_key.as_slice().try_into().unwrap()); + let sspk = Scalar::from_bytes(sending_signed_private_key.as_slice().try_into().unwrap()); + let ridk = CompressedEdwardsY(receiving_identity_key.as_slice().try_into().unwrap()).decompress(); + let repk = CompressedEdwardsY(receiving_ephemeral_key.as_slice().try_into().unwrap()).decompress(); + + if ridk.is_none().into() { + return "invalid receiving identity public key".to_string(); + } + + if repk.is_none().into() { + return "invalid receiving signed public key".to_string(); + } + + match x3dh::receiver_x3dh(&sidk, &sspk, &ridk.unwrap(), &repk.unwrap(), session_key_length) { + Some(result) => { + return format!("\"{}\"", BASE64_STANDARD.encode(result)); + } + None => { + return "could not perform key agreement".to_string(); + } + } +} + pub fn new_double_ratchet(session_key: &Vec, sending_header_key: &Vec, next_receiving_header_key: &Vec, is_sender: bool, sending_ephemeral_private_key: &Vec, receiving_ephemeral_key: &Vec) -> String { if sending_ephemeral_private_key.len() != 56 { return "invalid private key length".to_string(); @@ -64,7 +144,7 @@ pub fn new_double_ratchet(session_key: &Vec, sending_header_key: &Vec, n &session_key, &sending_header_key, &next_receiving_header_key, - true, + is_sender, sending_key, receiving_key.unwrap(), ); @@ -582,6 +662,44 @@ pub fn triple_ratchet_decrypt(ratchet_state_and_envelope: TripleRatchetStateAndE }; } +pub fn triple_ratchet_resize(ratchet_state: String, other: String, id: usize, total: usize) -> Vec> { + let tr = TripleRatchetParticipant::from_json(&ratchet_state); + if tr.is_err() { + return vec![vec![1]]; + } + + let other_bytes = hex::decode(other); + if other_bytes.is_err() { + return vec![other_bytes.unwrap_err().to_string().as_bytes().to_vec()]; + } + + let result = tr.unwrap().ratchet_resize(other_bytes.unwrap(), id, total); + if result.is_err() { + return vec![result.unwrap_err().to_string().as_bytes().to_vec()]; + } + + return result.unwrap(); +} + +pub fn triple_ratchet_verify_point(ratchet_state: String, point: String, id: usize) -> Result> { + let tr = TripleRatchetParticipant::from_json(&ratchet_state); + if tr.is_err() { + return Err(tr.unwrap_err()); + } + + let point_bytes = hex::decode(point); + if point_bytes.is_err() { + return Err(Box::new(point_bytes.unwrap_err())); + } + + let result = tr.unwrap().point_verify(point_bytes.unwrap(), id); + if result.is_err() { + return Err(result.unwrap_err()); + } + + return Ok(result.unwrap()); +} + #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/crates/channel/src/protocols/feldman.rs b/crates/channel/src/protocols/feldman.rs index c756523..4ca86b2 100644 --- a/crates/channel/src/protocols/feldman.rs +++ b/crates/channel/src/protocols/feldman.rs @@ -16,7 +16,7 @@ pub enum FeldmanError { CryptoError(String), } -#[derive(Clone, Copy, PartialEq)] +#[derive(Clone, Copy, PartialEq, Debug)] enum FeldmanRound { Uninitialized, Initialized, @@ -25,6 +25,7 @@ enum FeldmanRound { Reconstructed, } +#[derive(Debug)] pub struct Feldman { threshold: usize, total: usize, @@ -226,25 +227,13 @@ impl Feldman { return Err(FeldmanError::WrongRound); } - let mut coeffs = vec![self.secret]; - - for _ in 1..self.threshold { - coeffs.push(Scalar::random(rng)); - } + let samples = Feldman::construct_polynomial_samples(rng, self.secret, self.threshold, self.total); for i in 1..=self.total { - let mut result = coeffs[0]; - let x = Scalar::from(i as u32); - - for j in 1..self.threshold { - let term = coeffs[j] * Scalar::from(i.pow(j as u32) as u32); - result += term; - } - if i == self.id { - self.scalar = Some(result); + self.scalar = Some(samples[i-1]); } else { - self.frags_for_counterparties.insert(i, result.to_bytes().to_vec()); + self.frags_for_counterparties.insert(i, samples[i-1].to_bytes().to_vec()); } } @@ -252,6 +241,29 @@ impl Feldman { Ok(()) } + fn construct_polynomial_samples(rng: &mut R, secret: Scalar, threshold: usize, total: usize) -> Vec { + let mut coeffs = vec![secret]; + + for _ in 1..threshold { + coeffs.push(Scalar::random(rng)); + } + + let mut samples = Vec::::new(); + for i in 1..=total { + let mut result = coeffs[0]; + let x = Scalar::from(i as u32); + + for j in 1..threshold { + let term = coeffs[j] * Scalar::from(i.pow(j as u32) as u32); + result += term; + } + + samples.push(result); + } + + return samples; + } + pub fn scalar(&self) -> Option<&Scalar> { self.scalar.as_ref() } @@ -481,5 +493,51 @@ impl Feldman { pub fn public_key_bytes(&self) -> Vec { self.public_key.to_bytes().to_vec() } -} + pub fn redistribute(rng: &mut R, shares: Vec>, ids: &[usize], threshold: usize, total: usize) -> Result>, FeldmanError> { + if shares.len() != ids.len() { + return Err(FeldmanError::InvalidData("mismatch of shares and ids len".to_string())); + } + + let mut points = HashMap::::new(); + for (i, share) in shares.iter().enumerate() { + let point = Scalar::from_bytes(&(*share).clone().try_into().unwrap()); + if point.is_zero().into() { + return Err(FeldmanError::InvalidData(format!("invalid pubkey for {}", ids[i]).to_string())); + } + + points.insert(ids[i], point); + } + + let mut reconstructed_sum = Scalar::ZERO; + + for j in ids { + let mut num = Scalar::ONE; + let mut den = Scalar::ONE; + + for k in ids { + if j != k { + let j_scalar = Scalar::from(*j as u32); + let k_scalar = Scalar::from(*k as u32); + + num *= k_scalar; + den *= k_scalar - j_scalar; + } + } + + let den_inv = den.invert(); + let reconstructed_fragment = points[&j] * (num * den_inv); + reconstructed_sum += reconstructed_fragment; + } + + return Ok(Feldman::construct_polynomial_samples(rng, reconstructed_sum, threshold, total).iter().map(|s| s.to_bytes().to_vec()).collect()) + } + + pub fn get_scalar(&self) -> Scalar { + return self.scalar.unwrap(); + } + + pub fn get_id(&self) -> usize { + return self.id; + } +} diff --git a/crates/channel/src/protocols/tripleratchet.rs b/crates/channel/src/protocols/tripleratchet.rs index 85eb169..aa9e00c 100644 --- a/crates/channel/src/protocols/tripleratchet.rs +++ b/crates/channel/src/protocols/tripleratchet.rs @@ -48,6 +48,7 @@ pub struct PeerInfo { pub(crate) signed_pre_public_key: Vec, } +#[derive(Debug)] pub struct TripleRatchetParticipant { peer_key: Scalar, sending_ephemeral_private_key: Scalar, @@ -694,6 +695,63 @@ impl TripleRatchetParticipant { Ok((plaintext, should_dkg_ratchet)) } + pub fn ratchet_resize(&mut self, other: Vec, id: usize, total: usize) -> Result>, Box> { + if !self.async_dkg_ratchet { + return Err(Box::new(TripleRatchetError::CryptoError("cannot use non-async triple ratchet for resize".to_owned()))); + } + + if self.threshold != 2 { + return Err(Box::new(TripleRatchetError::CryptoError("cannot use larger threshold size than two for resize".to_owned()))); + } + + let rescaled = Feldman::redistribute(&mut OsRng, vec![self.dkg_ratchet.get_scalar().to_bytes().to_vec(), other], &vec![self.dkg_ratchet.get_id(), id], 2, total); + + if rescaled.is_err() { + return Err(Box::new(rescaled.unwrap_err())); + } + + return Ok(rescaled.unwrap()); + } + + + pub fn point_verify(&mut self, point: Vec, id: usize) -> Result> { + if !self.async_dkg_ratchet { + return Err(Box::new(TripleRatchetError::CryptoError("cannot use non-async triple ratchet for point verify".to_owned()))); + } + + if self.threshold != 2 { + return Err(Box::new(TripleRatchetError::CryptoError("cannot use larger threshold size than two for point verify".to_owned()))); + } + + if id == 0 { + return Err(Box::new(TripleRatchetError::CryptoError("invalid id".to_owned()))) + } + + let ours = (self.dkg_ratchet.get_scalar() * EdwardsPoint::generator()).compress(); + let ours_bytes = ours.as_bytes(); + + let mut shares = Vec::<&[u8]>::new(); + let mut ids = Vec::::new(); + + if self.dkg_ratchet.get_id() > id { + ids.push(id); + shares.push(&point); + ids.push(self.dkg_ratchet.get_id()); + shares.push(ours_bytes); + } else { + ids.push(self.dkg_ratchet.get_id()); + shares.push(ours_bytes); + ids.push(id); + shares.push(&point); + } + + let result = self.dkg_ratchet.combine_mul_share(shares, ids.as_slice()); + match result { + Ok(pubkey) => Ok(pubkey == self.dkg_ratchet.public_key_bytes()), + Err(e) => Err(Box::new(e)) + } + } + fn ratchet_sender_ephemeral_keys(&mut self) -> Result<(), Box> { let receiving_group_key = self.receiving_group_key.as_ref().ok_or_else(|| TripleRatchetError::CryptoError("Receiving group key not set".into()))?; self.sending_ephemeral_private_key = Scalar::random(&mut OsRng); @@ -823,16 +881,16 @@ impl TripleRatchetParticipant { let current_header_key = rkck[32..64].to_vec(); match self.decrypt(ciphertext, ¤t_header_key, None) { Ok(header) => Ok((header, true, true)), - Err(e) => Err(Box::new(e)), + Err(e) => Err(Box::new(TripleRatchetError::CryptoError(format!("header: {}", e.to_string())))), } } else { match self.decrypt(ciphertext, &self.next_header_key, None) { Ok(header) => Ok((header, true, false)), - Err(e) => Err(Box::new(e)), + Err(e) => Err(Box::new(TripleRatchetError::CryptoError(format!("header: {}", e.to_string())))), } } }, - Err(e) => Err(Box::new(e)), + Err(e) => Err(Box::new(TripleRatchetError::CryptoError(format!("header: {}", e.to_string())))), } } diff --git a/crates/ed448-rust/.editorconfig b/crates/ed448-rust/.editorconfig new file mode 100644 index 0000000..b175ed5 --- /dev/null +++ b/crates/ed448-rust/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +max_line_length = 120 +tab_width = 4 diff --git a/crates/ed448-rust/.github/workflows/audit.yml b/crates/ed448-rust/.github/workflows/audit.yml new file mode 100644 index 0000000..74f90da --- /dev/null +++ b/crates/ed448-rust/.github/workflows/audit.yml @@ -0,0 +1,28 @@ +name: Security audit + +on: + schedule: + # Launch everyday, at midnight + - cron: '0 0 * * *' + push: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + # Allow to manually trigger the check + workflow_dispatch: + +jobs: + security_audit: + runs-on: ubuntu-latest + + steps: + # Checkout Git repository + - name: Checkout sources + uses: actions/checkout@v2 + + # Security scan all dependencies, create a Pull Request if anyone found + - name: Security audit used dependencies + uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/crates/ed448-rust/.github/workflows/ci.yml b/crates/ed448-rust/.github/workflows/ci.yml new file mode 100644 index 0000000..f50a993 --- /dev/null +++ b/crates/ed448-rust/.github/workflows/ci.yml @@ -0,0 +1,171 @@ +name: Ci + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: + - main + - dev + pull_request: + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + check: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout sources + uses: actions/checkout@v2 + + # Install stable toolchain + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Runs the compilation checks + - name: Run cargo check with defaults feature + uses: actions-rs/cargo@v1 + with: + command: check + + # - name: Run cargo check with all features + # uses: actions-rs/cargo@v1 + # with: + # command: check + # args: --all-features + + clippy_and_fmt: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout sources + uses: actions/checkout@v2 + + # Install nightly toolchain + - name: Install nightly toolchain + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: nightly + override: true + components: clippy, rustfmt + + # Checks Rust code formatting + - name: Run cargo fmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + # Checks Rust code with Clippy + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + continue-on-error: true # note: the compiler unexpectedly panicked. this is a bug. + with: + command: clippy + args: --all-features -- -D warnings + + test: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout sources + uses: actions/checkout@v2 + + # Install nightly toolchain + - name: Install nightly toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + override: true + components: llvm-tools-preview + + # Build the binary + - name: Build binary + uses: actions-rs/cargo@v1 + with: + command: build + args: --all-features + env: + RUSTFLAGS: "-Zinstrument-coverage" + + # Runs the units tests in Debug mode + - name: Execute tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features + env: + RUSTFLAGS: "-Zinstrument-coverage" + LLVM_PROFILE_FILE: "your_name-%p-%m.profraw" + RUST_LOG: "ed448_rust" + + - name: Install grcov + run: curl -L https://github.com/mozilla/grcov/releases/latest/download/grcov-linux-x86_64.tar.bz2 | tar jxf - + + - name: Gather coverage data + run: ./grcov $GITHUB_WORKSPACE --source-dir $GITHUB_WORKSPACE --llvm --commit-sha $GITHUB_SHA --binary-path $GITHUB_WORKSPACE/target/debug/ --output-type lcov --branch --ignore-not-existing --ignore "/*" --output-path $LCOV_DESTINATION --service-name $GITHUB_WORKFLOW + env: + LCOV_DESTINATION: ${{ runner.temp }}/lcov.info + + - name: Codecov upload + uses: codecov/codecov-action@v1 + with: + #token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: ${{ runner.temp }}/lcov.info + name: codecov-ed448-rust # optional + fail_ci_if_error: true # optional (default = false) + + build: + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout sources + uses: actions/checkout@v2 + + # Install stable toolchain + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + # Build the project + - name: Build the project + uses: actions-rs/cargo@v1 + with: + command: build + args: --release diff --git a/crates/ed448-rust/.gitignore b/crates/ed448-rust/.gitignore new file mode 100644 index 0000000..a1a6602 --- /dev/null +++ b/crates/ed448-rust/.gitignore @@ -0,0 +1,4 @@ +/target +Cargo.lock +default.profraw +.DS_Store diff --git a/crates/ed448-rust/CHANGELOG.md b/crates/ed448-rust/CHANGELOG.md new file mode 100644 index 0000000..f5bb54b --- /dev/null +++ b/crates/ed448-rust/CHANGELOG.md @@ -0,0 +1,14 @@ +## v0.1.2 - upcoming + +* A feature was added to include [README.md](./README.md) during the tests to check examples in the usage. + + +## v0.1.1 + +* Add clippy directives +* Fix some documentation +* Add better documentation for the `PublicKey` struct + +## v0.1.0 + +Initial release \ No newline at end of file diff --git a/crates/ed448-rust/Cargo.toml b/crates/ed448-rust/Cargo.toml new file mode 100644 index 0000000..52cc4a2 --- /dev/null +++ b/crates/ed448-rust/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "ed448-rust" +version = "0.1.2" +authors = ["Lolo_32 "] +edition = "2018" +description = "Implementation of Edwards-Curve Digital Signature Algorithm (EdDSA) for ed448 only." +license = "MIT/Apache-2.0" +repository = "https://github.com/lolo32/ed448-rust" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +docinclude = [] # Used only for activating `doc(include="...")` on stable. + +[dependencies.lazy_static] +version = "1.4.0" + +[dependencies.num-bigint] +version = "0.4.0" + +[dependencies.num-integer] +version = "0.1.44" + +[dependencies.num-traits] +version = "0.2.14" + +[dependencies.opaque-debug] +version = "0.3.0" + +[dependencies.rand_core] +version = "0.6.2" +default-features = false +features = ["alloc"] + +[dependencies.sha3] +version = "0.9.1" + +[dependencies.subtle] +version = "2.4.0" +default-features = false +features = ["std"] + +[dev-dependencies.base64] +version = "0.13.0" + +[dev-dependencies.hex] +version = "0.4.3" + +[dev-dependencies.rand_core] +version = "0.6.2" +features = ["getrandom"] + +[package.metadata.docs.rs] +rustc-args = ["--cfg", "docsrs"] +features = ["docinclude"] # Activate `docinclude` during docs.rs build. diff --git a/crates/ed448-rust/LICENSE_APACHE2.txt b/crates/ed448-rust/LICENSE_APACHE2.txt new file mode 100644 index 0000000..67db858 --- /dev/null +++ b/crates/ed448-rust/LICENSE_APACHE2.txt @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/crates/ed448-rust/LICENSE_MIT.txt b/crates/ed448-rust/LICENSE_MIT.txt new file mode 100644 index 0000000..6f57f3e --- /dev/null +++ b/crates/ed448-rust/LICENSE_MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Lolo_32 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/ed448-rust/README.md b/crates/ed448-rust/README.md new file mode 100644 index 0000000..88e4c48 --- /dev/null +++ b/crates/ed448-rust/README.md @@ -0,0 +1,68 @@ +# Ed448-Rust + +[![Ci](https://github.com/lolo32/ed448-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/lolo32/ed448-rust/actions/workflows/ci.yml) +[![Security audit](https://github.com/lolo32/ed448-rust/actions/workflows/audit.yml/badge.svg)](https://github.com/lolo32/ed448-rust/actions/workflows/audit.yml) +[![codecov](https://codecov.io/gh/lolo32/ed448-rust/branch/main/graph/badge.svg?token=V206OZ48AA)](https://codecov.io/gh/lolo32/ed448-rust) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +[![Docs.rs](https://docs.rs/ed448-rust/badge.svg)](https://docs.rs/ed448-rust/) +[![Crates.io](https://img.shields.io/crates/v/ed448-rust)](https://crates.io/crates/ed448-rust) + +This is an implementation of Edwards-Curve Digital Signature Algorithm (EdDSA) +from the [RFC8032](https://tools.ietf.org/html/rfc8032) in pure Rust, +but only the ed448 support is implemented. + +It's a EdDSA for ed448 signing/verifying. + +_This is direct port of the Python code in the RFC, so it's the same warning +as it:_ + +> _**Note: This code is not intended for production. Although it should**_ +> _**produce correct results for every input, it is slow and makes no**_ +> _**attempt to avoid side-channel attacks.**_ + +## Usage + +```rust +use core::convert::TryFrom; +use rand_core::OsRng; +use ed448_rust::{PrivateKey, PublicKey}; + +fn main () { + // Generate a new random private key + let private_key = PrivateKey::new(&mut OsRng); + + // Store the key + let pkey_stored = private_key.as_bytes(); + + // Load a stored key before using it, or generating the public key + let private_key = PrivateKey::try_from(pkey_stored).unwrap(); + + // Extract associated public key + let public_key = PublicKey::from(&private_key); + + // Store the public key + let pubkey_stored = public_key.as_byte(); + + // Sign a message without context + let signature = private_key.sign(b"Message to sign", None).unwrap(); + // Sign a message with a context + let signature_ctx = private_key.sign(b"Message to sign", Some(&[0x01, 0xA6])).unwrap(); + // Sign a pre-hashed message without context + let signature_ph = private_key.sign_ph(b"Message to sign", None).unwrap(); + + // Verify the signature without context + assert!(public_key.verify(b"Message to sign", &signature, None).is_ok()); + // Verify the signature with context + assert!(public_key.verify(b"Message to sign", &signature_ctx, Some(&[0x01, 0xA6])).is_ok()); + // Verify the signature with the pre-hash and without context + assert!(public_key.verify_ph(b"Message to sign", &signature_ph, None).is_ok()); +} +``` + +## License + +This code is licensed under [MIT] / [Apache2.0] + +[MIT]: LICENSE_MIT.txt +[Apache2.0]: LICENSE_APACHE2.txt diff --git a/crates/ed448-rust/src/error.rs b/crates/ed448-rust/src/error.rs new file mode 100644 index 0000000..2b82fe5 --- /dev/null +++ b/crates/ed448-rust/src/error.rs @@ -0,0 +1,46 @@ +// Copyright 2021 Lolo_32 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Errors of this crate +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Ed448Error { + /// The provided array is not in the correct length for the private key. + /// + /// It must be [`crate::KEY_LENGTH`]. + /// + /// See [PrivateKey::from](crate::PrivateKey::from). + WrongKeyLength, + /// The provided array is not in the correct length for the public key. + /// + /// It must be [`crate::KEY_LENGTH`]. + /// + /// See [PublicKey::from](crate::PublicKey::from). + WrongPublicKeyLength, + /// The provided array is not in the correct length for the signature. + /// + /// It must be [`SIG_LENGTH`](crate::SIG_LENGTH). + /// + /// See [PublicKey::verify](crate::PublicKey::verify). + WrongSignatureLength, + /// The computed point is not valid (maybe forged/altered public key or signature). + InvalidPoint, + /// Signature verification failed. + /// + /// See [PublicKey::verify](crate::PublicKey::verify). + InvalidSignature, + /// The provided context byte array is too long. + /// + /// It must not be more than 256 byte. + ContextTooLong, +} diff --git a/crates/ed448-rust/src/lib.rs b/crates/ed448-rust/src/lib.rs new file mode 100644 index 0000000..93cc579 --- /dev/null +++ b/crates/ed448-rust/src/lib.rs @@ -0,0 +1,291 @@ +// Copyright 2021 Lolo_32 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg_attr(feature = "docinclude", feature(external_doc))] +#![deny( + missing_docs, + missing_copy_implementations, + missing_debug_implementations, + trivial_numeric_casts, + unreachable_pub, + unsafe_code, + unused_extern_crates, + unused_qualifications, + single_use_lifetimes, + unused_import_braces, + unused_lifetimes, + unused_results, + clippy::all, + clippy::pedantic, + clippy::nursery +)] +#![doc( + test(no_crate_inject, attr(deny(warnings))), + test(attr(allow(unused_variables))), + html_no_source +)] +#![deny( + clippy::absurd_extreme_comparisons, + clippy::almost_swapped, + clippy::approx_constant, + clippy::async_yields_async, + clippy::bad_bit_mask, + clippy::cast_ref_to_mut, + clippy::clone_double_ref, + clippy::cmp_nan, + clippy::deprecated_semver, + clippy::derive_hash_xor_eq, + clippy::derive_ord_xor_partial_ord, + clippy::drop_copy, + clippy::drop_ref, + clippy::enum_clike_unportable_variant, + clippy::eq_op, + clippy::erasing_op, + clippy::float_cmp, + clippy::float_equality_without_abs, + clippy::fn_address_comparisons, + clippy::for_loops_over_fallibles, + clippy::forget_copy, + clippy::forget_ref, + clippy::if_let_mutex, + clippy::if_same_then_else, + clippy::ifs_same_cond, + clippy::ineffective_bit_mask, + clippy::infinite_iter, + clippy::inherent_to_string_shadow_display, + clippy::inline_fn_without_body, + clippy::invalid_atomic_ordering, + clippy::invalid_regex, + clippy::invisible_characters, + clippy::iter_next_loop, + clippy::iterator_step_by_zero, + clippy::let_underscore_lock, + clippy::logic_bug, + clippy::mem_discriminant_non_enum, + clippy::mem_replace_with_uninit, + clippy::min_max, + clippy::mismatched_target_os, + clippy::mistyped_literal_suffixes, + clippy::modulo_one, + clippy::mut_from_ref, + clippy::mutable_key_type, + clippy::never_loop, + clippy::nonsensical_open_options, + clippy::not_unsafe_ptr_arg_deref, + clippy::option_env_unwrap, + clippy::out_of_bounds_indexing, + clippy::panicking_unwrap, + clippy::possible_missing_comma, + clippy::reversed_empty_ranges, + clippy::self_assignment, + clippy::serde_api_misuse, + clippy::size_of_in_element_count, + clippy::suspicious_arithmetic_impl, + clippy::suspicious_op_assign_impl, + clippy::to_string_in_display, + clippy::transmuting_null, + clippy::undropped_manually_drops, + clippy::uninit_assumed_init, + clippy::unit_cmp, + clippy::unit_return_expecting_ord, + clippy::unsound_collection_transmute, + clippy::unused_io_amount, + clippy::useless_attribute, + clippy::vec_resize_to_zero, + clippy::vtable_address_comparisons, + clippy::while_immutable_condition, + clippy::wrong_transmute, + clippy::zst_offset +)] +#![allow( + non_snake_case, + non_upper_case_globals, + clippy::similar_names, + clippy::module_name_repetitions +)] + +//! # EdDSA implementation for ed448 +//! +//! This is a Edwards-Curve Digital Signature Algorithm (EdDSA) for ed448 only +//! in pure rust. +//! +//! # Usage +//! +//! There is two variants that can be combined to sign/verify: +//! +//! 1. [`PrivateKey::sign`](crate::PrivateKey::sign) to sign all the content +//! as-it and [`PrivateKey::sign_ph`](crate::PrivateKey::sign_ph) to +//! pre-hash the message internaly before signing it. It will be hashed +//! using Shake256 and the result of 64 byte will be signed/verified. +//! +//! Note: use the same variant for verifying the signature. +//! +//! 2. The second parameter of [`sign`](crate::PrivateKey::sign)/ +//! [`sign_ph`](crate::PrivateKey::sign_ph) and the third of +//! [`verify`](crate::PublicKey::verify)/ +//! [`verify_ph`](crate::PublicKey::verify_ph) if an optional context +//! of 255 byte length max. +//! +//! The context can be used to facilitate different signature over +//! different protocol, but it must be immuable over the protocol. +//! More information about this can be found at +//! [RFC 8032 Use of Contexts](https://tools.ietf.org/html/rfc8032#section-8.3). +//! +//! # Examples +//! +//! ## Generating a new key pair +//! +//! ``` +//! use rand_core::OsRng; +//! use ed448_rust::{PrivateKey, PublicKey}; +//! let private_key = PrivateKey::new(&mut OsRng); +//! let public_key = PublicKey::from(&private_key); +//! ``` +//! +//! ## Sign a message +//! +//! ``` +//! # use rand_core::OsRng; +//! use ed448_rust::{PrivateKey, Ed448Error}; +//! # let retrieve_pkey = || PrivateKey::new(&mut OsRng); +//! let message = b"Message to sign"; +//! let private_key = retrieve_pkey(); +//! match private_key.sign(message, None) { +//! Ok(signature) => { +//! // Signature OK, use it +//! // This is a slice of 144 byte length +//! } +//! Err(Ed448Error::ContextTooLong) => { +//! // The used context is more than 255 bytes length +//! } +//! Err(_) => unreachable!() +//! } +//! ``` +//! +//! ## Verify a signature +//! +//! ``` +//! # use rand_core::OsRng; +//! use ed448_rust::{PublicKey, Ed448Error}; +//! # let private_key = ed448_rust::PrivateKey::new(&mut OsRng); +//! let message = b"Signed message to verify"; +//! # let retrieve_signature = || private_key.sign(message, None).unwrap(); +//! # let retrieve_pubkey = || PublicKey::from(&private_key); +//! let public_key = retrieve_pubkey(); // A slice or array of KEY_LENGTH byte length +//! let signature = retrieve_signature(); // A slice or array of SIG_LENGTH byte length +//! match public_key.verify(message, &signature, None) { +//! Ok(()) => { +//! // Signature OK, use the message +//! } +//! Err(Ed448Error::InvalidSignature) => { +//! // The verification of the signature is invalid +//! } +//! Err(Ed448Error::ContextTooLong) => { +//! // The used context is more than 255 bytes length +//! } +//! Err(Ed448Error::WrongSignatureLength) => { +//! // The signature is not 144 bytes length +//! } +//! Err(_) => unreachable!() +//! } +//! ``` + +use sha3::{ + digest::{ExtendableOutput, Update}, + Shake256, +}; + +pub use crate::error::Ed448Error; + +pub use private_key::PrivateKey; +pub use public_key::PublicKey; +use std::borrow::Cow; + +mod error; +mod point; +mod private_key; +mod public_key; + +#[allow( + missing_docs, + missing_copy_implementations, + missing_debug_implementations +)] +#[doc(hidden)] +#[cfg_attr(feature = "docinclude", doc(include = "../README.md"))] +pub struct ReadmeDoctests; + +/// Specialized [`Result`](core::result::Result) for this crate. +pub type Result = core::result::Result; + +/// Length of either a public or a private key length in byte. +pub const KEY_LENGTH: usize = 57; +/// Length of the signature length in byte. +pub const SIG_LENGTH: usize = 114; + +/// Indicate if the message need to be pre-hashed before being signed/verified +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +enum PreHash { + /// Pre-hash the message + True, + /// Leave the message unchanged + False, +} + +impl From for u8 { + #[inline] + fn from(hash: PreHash) -> Self { + match hash { + PreHash::False => 0, + PreHash::True => 1, + } + } +} + +/// Produce a Shake256 for signing/verifying signatures +fn shake256(items: Vec<&[u8]>, ctx: &[u8], pre_hash: PreHash) -> Box<[u8]> { + #[allow(clippy::cast_possible_truncation)] + let mut shake = Shake256::default() + .chain(b"SigEd448") + .chain(&[pre_hash.into(), ctx.len() as u8]) + .chain(ctx); + for item in items { + shake.update(item); + } + shake.finalize_boxed(114) +} + +/// Common tasks for signing/verifying +#[allow(clippy::type_complexity)] +fn init_sig<'a, 'b>( + ctx: Option<&'b [u8]>, + pre_hash: PreHash, + msg: &'a [u8], +) -> Result<(Cow<'b, [u8]>, Cow<'a, [u8]>)> { + let ctx = ctx.unwrap_or(b""); + if ctx.len() > 255 { + return Err(Ed448Error::ContextTooLong); + } + let ctx = Cow::Borrowed(ctx); + + let msg = match pre_hash { + PreHash::False => Cow::Borrowed(msg), + PreHash::True => { + let hash = Shake256::default().chain(msg).finalize_boxed(64).to_vec(); + Cow::Owned(hash) + } + }; + + Ok((ctx, msg)) +} diff --git a/crates/ed448-rust/src/point.rs b/crates/ed448-rust/src/point.rs new file mode 100644 index 0000000..e952fe7 --- /dev/null +++ b/crates/ed448-rust/src/point.rs @@ -0,0 +1,433 @@ +// Copyright 2021 Lolo_32 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::{ + convert::TryInto, + ops::{Add, Div, Mul, Neg, Sub}, +}; + +use lazy_static::lazy_static; +use num_bigint::{BigInt, Sign}; +use num_traits::{One, Zero}; + +use crate::{Ed448Error, KEY_LENGTH}; +use subtle::{Choice, ConstantTimeEq}; + +lazy_static! { + // 2 ^ 448 - 2 ^224 - 1 + static ref p: BigInt = BigInt::from(2).pow(448).sub(BigInt::from(2).pow(224)) - 1; + static ref d: Field = Field::new(BigInt::from(-39081)); + static ref f0: Field = Field::new(BigInt::zero()); + static ref f1: Field = Field::new(BigInt::one()); + static ref xb: Field = Field::new(BigInt::from_bytes_be( + Sign::Plus, + &[ + 0x4F, 0x19, 0x70, 0xC6, 0x6B, 0xED, 0x0D, 0xED, 0x22, 0x1D, 0x15, 0xA6, 0x22, 0xBF, + 0x36, 0xDA, 0x9E, 0x14, 0x65, 0x70, 0x47, 0x0F, 0x17, 0x67, 0xEA, 0x6D, 0xE3, 0x24, + 0xA3, 0xD3, 0xA4, 0x64, 0x12, 0xAE, 0x1A, 0xF7, 0x2A, 0xB6, 0x65, 0x11, 0x43, 0x3B, + 0x80, 0xE1, 0x8B, 0x00, 0x93, 0x8E, 0x26, 0x26, 0xA8, 0x2B, 0xC7, 0x0C, 0xC0, 0x5E, + ] + )); + static ref yb: Field = Field::new(BigInt::from_bytes_be( + Sign::Plus, + &[ + 0x69, 0x3F, 0x46, 0x71, 0x6E, 0xB6, 0xBC, 0x24, 0x88, 0x76, 0x20, 0x37, 0x56, 0xC9, + 0xC7, 0x62, 0x4B, 0xEA, 0x73, 0x73, 0x6C, 0xA3, 0x98, 0x40, 0x87, 0x78, 0x9C, 0x1E, + 0x05, 0xA0, 0xC2, 0xD7, 0x3A, 0xD3, 0xFF, 0x1C, 0xE6, 0x7C, 0x39, 0xC4, 0xFD, 0xBD, + 0x13, 0x2C, 0x4E, 0xD7, 0xC8, 0xAD, 0x98, 0x08, 0x79, 0x5B, 0xF2, 0x30, 0xFA, 0x14, + ] + )); + + static ref l: BigInt = BigInt::from_bytes_be( + Sign::Plus, + &[ + 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7c, 0xca, 0x23, 0xe9, 0xc4, 0x4e, 0xdb, 0x49, 0xae, 0xd6, 0x36, 0x90, 0x21, 0x6c, + 0xc2, 0x72, 0x8d, 0xc5, 0x8f, 0x55, 0x23, 0x78, 0xc2, 0x92, 0xab, 0x58, 0x44, 0xf3, + ] + ); +} + +#[derive(Debug, Clone)] +pub struct Field(BigInt); + +impl Field { + pub fn new(value: BigInt) -> Self { + if value < BigInt::zero() { + Self((&p as &BigInt) + value) + } else { + Self(value % &p as &BigInt) + } + } + + /// Field inverse (inverse of 0 is 0). + #[inline] + pub fn inv(self) -> Self { + Self::new(self.0.modpow(&(&p as &BigInt - 2), &p)) + } + + /// Compute sign of number, 0 or 1. The sign function + /// has the following property: + /// sign(x) = 1 - sign(-x) if x != 0. + #[inline] + pub fn sign(&self) -> BigInt { + &self.0 % 2 + } + + /// Field square root. Returns none if square root does not exist. + /// Note: not presently implemented for p mod 8 = 1 case. + pub fn sqrt(self) -> crate::Result { + // Compute candidate square root. + let y = self + .0 + .modpow(&((&p as &BigInt).add(1_u32).div(&4)), &p as &BigInt); + let y = Self::new(y); + // Check square root candidate valid. + if &y * &y == self { + Ok(y) + } else { + Err(Ed448Error::InvalidPoint) + } + } + + /// Is the field element the additive identity? + #[inline] + pub fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +impl PartialEq for Field { + fn eq(&self, other: &Self) -> bool { + fn sign_to_choice(sign: Sign) -> Choice { + match sign { + Sign::Plus => 1, + Sign::Minus => 0, + Sign::NoSign => unreachable!(), + } + .into() + } + + let me = self.0.to_u64_digits(); + let other = other.0.to_u64_digits(); + let val = me.1.ct_eq(&other.1); + let sign_me = sign_to_choice(me.0); + let sign_other = sign_to_choice(other.0); + let sign = sign_me ^ sign_other; + (val & !sign).into() + } +} + +impl Add for Field { + type Output = Self; + + #[inline] + fn add(self, other: Self) -> Self { + self + &other + } +} + +impl Add<&'_ Self> for Field { + type Output = Self; + + #[inline] + fn add(self, rhs: &Self) -> Self { + Self::new(self.0 + &rhs.0) + } +} + +impl Add<&'_ Field> for &'_ Field { + type Output = Field; + + #[inline] + fn add(self, other: &Field) -> Self::Output { + self.clone() + other + } +} + +impl Sub for Field { + type Output = Self; + + #[inline] + fn sub(self, other: Self) -> Self { + self - &other + } +} + +impl Sub<&'_ Self> for Field { + type Output = Self; + + #[inline] + fn sub(self, other: &Self) -> Self { + Self::new(self.0 + &p as &BigInt - &other.0) + } +} + +impl Sub<&'_ Field> for &'_ Field { + type Output = Field; + + #[inline] + fn sub(self, other: &Field) -> Field { + self.clone() - other + } +} + +impl Mul for Field { + type Output = Self; + + #[inline] + fn mul(self, other: Self) -> Self { + self * &other + } +} + +impl Mul<&'_ Self> for Field { + type Output = Self; + + #[inline] + fn mul(self, other: &Self) -> Self { + Self::new(self.0 * &other.0) + } +} + +impl Mul<&'_ Field> for &'_ Field { + type Output = Field; + + #[inline] + fn mul(self, other: &Field) -> Field { + self.clone() * other + } +} + +impl Neg for Field { + type Output = Self; + + #[inline] + fn neg(self) -> Self { + Self::new(&p as &BigInt - self.0) + } +} + +impl Div for Field { + type Output = Self; + + #[inline] + fn div(self, other: Self) -> Self { + self / &other + } +} + +#[allow(clippy::suspicious_arithmetic_impl)] +impl Div<&'_ Self> for Field { + type Output = Self; + + #[inline] + fn div(self, other: &Self) -> Self { + self * other.clone().inv() + } +} + +impl Div<&'_ Field> for &'_ Field { + type Output = Field; + + #[inline] + fn div(self, other: &'_ Field) -> Field { + self.clone() / other + } +} + +#[derive(Debug, Clone)] +pub struct Point { + x: Field, + y: Field, + z: Field, +} + +impl Point { + pub fn new(x: &Field, y: &Field) -> crate::Result { + // Check that the point is actually on the curve. + if y * y + x * x == (&f1 as &Field) + &((&d as &Field) * x * x * y * y) { + Ok(Self { + x: x.clone(), + y: y.clone(), + ..Self::default() + }) + } else { + Err(Ed448Error::InvalidPoint) + } + } + + /// Order of basepoint. + #[inline] + pub fn l() -> &'static BigInt { + &l as &BigInt + } + + #[inline] + pub fn new_stdbase() -> Self { + Self::new(&f0, &f1).unwrap() + } + + /// Point doubling. + pub fn double(self) -> Self { + // The formulas are from EFD. + let (x1s, y1s, z1s) = (&self.x * &self.x, &self.y * &self.y, &self.z * &self.z); + let xys = &self.x + &self.y; + let F = &x1s + &y1s; + let J = &F - &(&z1s + &z1s); + let (x, y, z) = ( + (&xys * &xys - &x1s - &y1s) * &J, + &F * &(&x1s - &y1s), + &F * &J, + ); + + Self { x, y, z } + } + + /// Encode a point representation. + pub fn encode(&self) -> [u8; KEY_LENGTH] { + let (xp, yp) = (&self.x / &self.z, &self.y / &self.z); + + // Encode y. + let mut tmp = yp.0.magnitude().to_bytes_le(); + tmp.resize_with(KEY_LENGTH, Default::default); + let mut s: [u8; KEY_LENGTH] = tmp.try_into().unwrap(); + + // Add sign bit of x to encoding. + if !xp.sign().is_zero() { + s[56] |= 0b1000_0000; + } + s + } + + /// Decode a point representation. + pub fn decode(s: &[u8]) -> crate::Result { + // Extract signbit. + let xs = BigInt::from(s[56] >> 7); + // Decode y. If this fails, fail. + let y = Self::frombytes(s)?; + // Try to recover x. If it does not exist, or if zero and xs + // are wrong, fail. + let mut x = Self::solve_x2(&y).sqrt()?; + if x.is_zero() && xs != x.sign() { + return Err(Ed448Error::InvalidPoint); + } + // If sign of x isn't correct, flip it. + if x.sign() != xs { + x = -x; + } + // Return the constructed point. + Self::new(&x, &y) + } + + /// Unserialize number from bits. + fn frombytes(x: &[u8]) -> crate::Result { + let rv = BigInt::from_bytes_le(Sign::Plus, x) % BigInt::from(2).pow(455); + if &rv < &p as &BigInt { + Ok(Field::new(rv)) + } else { + Err(Ed448Error::InvalidPoint) + } + } + + /// Solve for x^2. + #[inline] + fn solve_x2(y: &Field) -> Field { + (y * y - &f1 as &Field) / (&d as &Field * y * y - &f1 as &Field) + } +} + +impl Mul<&'_ BigInt> for Point { + type Output = Self; + + #[inline] + fn mul(self, x: &BigInt) -> Self { + self * x.clone() + } +} + +impl Mul for Point { + type Output = Self; + + fn mul(mut self, mut x: BigInt) -> Self { + let mut r = Self::new_stdbase(); + while !x.is_zero() { + if !((&x % 2) as BigInt).is_zero() { + r = r + &self; + } + self = self.double(); + x /= 2; + } + r + } +} + +impl Add for Point { + type Output = Self; + + fn add(self, y: Self) -> Self { + // The formulas are from EFD. + let (xcp, ycp, zcp) = (&self.x * &y.x, &self.y * &y.y, &self.z * &y.z); + let B = &zcp * &zcp; + let E = &d as &Field * &xcp * &ycp; + let (F, G) = (&B - &E, B + E); + + let x = &zcp * &F * ((self.x + self.y) * (y.x + y.y) - &xcp - &ycp); + let (y, z) = (zcp * &G * (ycp - xcp), F * G); + + Self { x, y, z } + } +} + +impl Add<&'_ Self> for Point { + type Output = Self; + + #[inline] + fn add(self, other: &Self) -> Self { + self + other.clone() + } +} + +impl Add<&'_ Point> for &'_ Point { + type Output = Point; + + #[inline] + fn add(self, other: &Point) -> Point { + self.clone() + other.clone() + } +} + +impl PartialEq for Point { + fn eq(&self, other: &Self) -> bool { + // Need to check x1/z1 == x2/z2 and similarly for y, so cross + // multiply to eliminate divisions. + let xn1 = &self.x * &other.z; + let xn2 = &other.x * &self.z; + let yn1 = &self.y * &other.z; + let yn2 = &other.y * &self.z; + xn1 == xn2 && yn1 == yn2 + } +} + +impl Default for Point { + #[inline] + fn default() -> Self { + Self { + x: xb.clone(), + y: yb.clone(), + z: Field::new(BigInt::one()), + } + } +} diff --git a/crates/ed448-rust/src/private_key.rs b/crates/ed448-rust/src/private_key.rs new file mode 100644 index 0000000..5e2b238 --- /dev/null +++ b/crates/ed448-rust/src/private_key.rs @@ -0,0 +1,255 @@ +// Copyright 2021 Lolo_32 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::convert::{TryFrom, TryInto}; + +use num_bigint::{BigInt, Sign}; +use rand_core::{CryptoRng, RngCore}; +use sha3::{ + digest::{ExtendableOutput, Update}, + Shake256, +}; + +use crate::{ + init_sig, point::Point, shake256, Ed448Error, PreHash, PublicKey, KEY_LENGTH, SIG_LENGTH, +}; + +#[allow(clippy::redundant_pub_crate)] +pub(crate) type PrivateKeyRaw = [u8; KEY_LENGTH]; +#[allow(clippy::redundant_pub_crate)] +pub(crate) type SeedRaw = [u8; KEY_LENGTH]; + +/// This represent a private key. **Must be kept secret.** +/// +/// Could be used to generate a new one or restore an older already saved. +#[derive(Copy, Clone)] +pub struct PrivateKey(PrivateKeyRaw); + +opaque_debug::implement!(PrivateKey); + +impl PrivateKey { + /// Generate a random key. + /// + /// # Example + /// + /// ``` + /// use rand_core::OsRng; + /// use ed448_rust::PrivateKey; + /// let private_key = PrivateKey::new(&mut OsRng); + /// ``` + pub fn new(rnd: &mut T) -> Self + where + T: CryptoRng + RngCore, + { + let mut key = [0; KEY_LENGTH]; + rnd.fill_bytes(&mut key); + Self::from(key) + } + + /// Convert the private key to a format exportable. + /// + /// # Example + /// + /// ``` + /// # use rand_core::OsRng; + /// # use ed448_rust::PrivateKey; + /// # let private_key = PrivateKey::new(&mut OsRng); + /// let exportable_pkey = private_key.as_bytes(); + /// ``` + #[inline] + #[must_use] + pub const fn as_bytes(&self) -> &[u8; KEY_LENGTH] { + &self.0 + } + + pub(crate) fn expand(&self) -> (PrivateKeyRaw, SeedRaw) { + // 1. Hash the 57-byte private key using SHAKE256(x, 114), storing the + // digest in a 114-octet large buffer, denoted h. + let h = Shake256::default() + .chain(self.as_bytes()) + .finalize_boxed(114); + // Only the lower 57 bytes are used for generating the public key. + let mut s: [u8; KEY_LENGTH] = h[..KEY_LENGTH].try_into().unwrap(); + + // 2. Prune the buffer: The two least significant bits of the first + // octet are cleared, all eight bits the last octet are cleared, and + // the highest bit of the second to last octet is set. + s[0] &= 0b1111_1100; + s[56] = 0; + s[55] |= 0b1000_0000; + + let seed: [u8; KEY_LENGTH] = h[KEY_LENGTH..].try_into().unwrap(); + + (s, seed) + } + + /// Sign with key pair. + /// + /// It's possible to indicate a context. More information in + /// [RFC8032 8.3 Use of contexts](https://tools.ietf.org/html/rfc8032#section-8.3). + /// + /// # Examples + /// + /// Without any context. + /// + /// ``` + /// use ed448_rust::PrivateKey; + /// let pkey = PrivateKey::from([0xcd, 0x23, 0xd2, 0x4f, 0x71, 0x42, 0x74, 0xe7, 0x44, 0x34, 0x32, 0x37, 0xb9, + /// 0x32, 0x90, 0xf5, 0x11, 0xf6, 0x42, 0x5f, 0x98, 0xe6, 0x44, 0x59, 0xff, 0x20, 0x3e, 0x89, 0x85, + /// 0x08, 0x3f, 0xfd, 0xf6, 0x05, 0x00, 0x55, 0x3a, 0xbc, 0x0e, 0x05, 0xcd, 0x02, 0x18, 0x4b, 0xdb, + /// 0x89, 0xc4, 0xcc, 0xd6, 0x7e, 0x18, 0x79, 0x51, 0x26, 0x7e, 0xb3, 0x28]); + /// let msg = &[0x0c, 0x3e, 0x54, 0x40, 0x74, 0xec, 0x63, 0xb0, 0x26, 0x5e, 0x0c]; + /// let sig = pkey.sign(msg, None).unwrap(); + /// + /// assert_eq!( + /// sig.iter().map(|b| format!("{:02x}", b)).collect::>().concat(), + /// "1f0a8888ce25e8d458a21130879b840a9089d999aaba039eaf3e3afa090a09d389dba82c4ff2ae8a\ + /// c5cdfb7c55e94d5d961a29fe0109941e00b8dbdeea6d3b051068df7254c0cdc129cbe62db2dc9\ + /// 57dbb47b51fd3f213fb8698f064774250a5028961c9bf8ffd973fe5d5c206492b140e00" + /// ); + /// ``` + /// + /// With a context. + /// + /// ``` + /// use ed448_rust::PrivateKey; + /// let pkey = PrivateKey::from([0xc4, 0xea, 0xb0, 0x5d, 0x35, 0x70, 0x07, 0xc6, 0x32, 0xf3, 0xdb, 0xb4, 0x84, + /// 0x89, 0x92, 0x4d, 0x55, 0x2b, 0x08, 0xfe, 0x0c, 0x35, 0x3a, 0x0d, 0x4a, 0x1f, 0x00, 0xac, 0xda, + /// 0x2c, 0x46, 0x3a, 0xfb, 0xea, 0x67, 0xc5, 0xe8, 0xd2, 0x87, 0x7c, 0x5e, 0x3b, 0xc3, 0x97, 0xa6, + /// 0x59, 0x94, 0x9e, 0xf8, 0x02, 0x1e, 0x95, 0x4e, 0x0a, 0x12, 0x27, 0x4e]); + /// let msg = &[03]; + /// let sig = pkey.sign(msg, Some(&[0x66, 0x6f, 0x6f])).unwrap(); + /// + /// assert_eq!( + /// sig.iter().map(|b| format!("{:02x}", b)).collect::>().concat(), + /// "d4f8f6131770dd46f40867d6fd5d5055de43541f8c5e35abbcd001b32a89f7d2151f7647f11d8ca2\ + /// ae279fb842d607217fce6e042f6815ea000c85741de5c8da1144a6a1aba7f96de42505d7a7298\ + /// 524fda538fccbbb754f578c1cad10d54d0d5428407e85dcbc98a49155c13764e66c3c00" + /// ); + /// ``` + /// + /// # Errors + /// + /// * [`Ed448Error::ContextTooLong`] if the context is more than 255 byte length. + #[inline] + pub fn sign(&self, msg: &[u8], ctx: Option<&[u8]>) -> crate::Result<[u8; SIG_LENGTH]> { + self.sign_real(msg, ctx, PreHash::False) + } + + /// Sign with key pair. Message is pre-hashed before signed. + /// + /// The message is hashed before being signed. The size of the signed message in this + /// case is always 64 bytes length. + /// + /// See [`PrivateKey::sign`]. + /// + /// # Errors + /// + /// * [`Ed448Error::ContextTooLong`] if the context is more than 255 byte length. + #[inline] + pub fn sign_ph(&self, msg: &[u8], ctx: Option<&[u8]>) -> crate::Result<[u8; SIG_LENGTH]> { + self.sign_real(msg, ctx, PreHash::True) + } + + fn sign_real( + &self, + msg: &[u8], + ctx: Option<&[u8]>, + pre_hash: PreHash, + ) -> crate::Result<[u8; SIG_LENGTH]> { + let (ctx, msg) = init_sig(ctx, pre_hash, msg)?; + // Expand key. + let (a, seed) = &self.expand(); + let a = BigInt::from_bytes_le(Sign::Plus, a); + // Calculate r and R (R only used in encoded form). + let r = shake256(vec![seed, &msg], ctx.as_ref(), pre_hash); + let r = BigInt::from_bytes_le(Sign::Plus, r.as_ref()) % Point::l(); + let R = (Point::default() * &r).encode(); + // Calculate h. + let h = shake256( + vec![&R, &PublicKey::from(a.clone()).as_byte(), &msg], + ctx.as_ref(), + pre_hash, + ); + let h = BigInt::from_bytes_le(Sign::Plus, h.as_ref()) % Point::l(); + // Calculate s. + let S = (r + h * a) % Point::l(); + // The final signature is a concatenation of R and S. + let mut S = S.magnitude().to_bytes_le(); + S.resize_with(KEY_LENGTH, Default::default); + let S: [u8; KEY_LENGTH] = S.try_into().unwrap(); + + Ok([R, S].concat().try_into().unwrap()) + } +} + +/// Restore the private key from the slice. +impl From for PrivateKey { + #[inline] + fn from(array: PrivateKeyRaw) -> Self { + Self(array) + } +} + +/// Restore the private key from an array. +/// +/// # Error +/// +/// Could return [`Ed448Error::WrongKeyLength`] if the array's length +/// is not [`KEY_LENGTH`]. +impl TryFrom<&'_ [u8]> for PrivateKey { + type Error = Ed448Error; + + fn try_from(bytes: &[u8]) -> crate::Result { + if bytes.len() != KEY_LENGTH { + return Err(Ed448Error::WrongKeyLength); + } + let bytes: &[u8; KEY_LENGTH] = bytes.try_into().unwrap(); + Ok(Self::from(bytes)) + } +} + +impl From<&'_ PrivateKeyRaw> for PrivateKey { + #[inline] + fn from(bytes: &PrivateKeyRaw) -> Self { + Self::from(*bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rand_core::OsRng; + + #[test] + fn create_new_pkey() { + let pkey = PrivateKey::new(&mut OsRng); + let a = pkey.as_bytes(); + assert_eq!(a.len(), KEY_LENGTH); + } + + #[test] + fn invalid_key_len() { + let invalid_pk = PrivateKey::try_from(&[0x01_u8][..]); + assert_eq!(invalid_pk.unwrap_err(), Ed448Error::WrongKeyLength); + } + + #[test] + fn invalid_context_length() { + let pkey = PrivateKey::new(&mut OsRng); + let ctx = [0; 256]; + let invalid_sig = pkey.sign(b"message", Some(&ctx)); + assert_eq!(invalid_sig.unwrap_err(), Ed448Error::ContextTooLong); + } +} diff --git a/crates/ed448-rust/src/public_key.rs b/crates/ed448-rust/src/public_key.rs new file mode 100644 index 0000000..3574273 --- /dev/null +++ b/crates/ed448-rust/src/public_key.rs @@ -0,0 +1,293 @@ +// Copyright 2021 Lolo_32 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::convert::TryFrom; + +use num_bigint::{BigInt, Sign}; + +use crate::{ + init_sig, + point::Point, + private_key::PrivateKey, + {shake256, Ed448Error, PreHash, KEY_LENGTH, SIG_LENGTH}, +}; + +/// This is a public key. _Should be distributed._ +/// +/// You can extract a `PublicKey` by calling [`Self::from()`]. +#[derive(Clone)] +pub struct PublicKey(Point); + +opaque_debug::implement!(PublicKey); + +impl PublicKey { + /// Convert the public key to an easily exportable format. + #[inline] + #[must_use] + pub fn as_byte(&self) -> [u8; 57] { + // 4. The public key A is the encoding of the point [s]B. + self.0.encode() + } + + /// Verify signature with public key. + /// + /// # Example + /// + /// ``` + /// # use rand_core::OsRng; + /// use ed448_rust::{PublicKey, Ed448Error}; + /// # let private_key = ed448_rust::PrivateKey::new(&mut OsRng); + /// let message = b"Signed message to verify"; + /// # let retrieve_signature = || private_key.sign(message, None).unwrap(); + /// # let retrieve_pubkey = || PublicKey::from(&private_key); + /// let public_key = retrieve_pubkey(); + /// let signature = retrieve_signature(); + /// match public_key.verify(message, &signature, None) { + /// Ok(()) => { + /// // Signature OK, use the message + /// } + /// Err(Ed448Error::InvalidSignature) => { + /// // The verification of the signature is invalid + /// } + /// Err(Ed448Error::ContextTooLong) => { + /// // The used context is more than 255 bytes length + /// } + /// Err(Ed448Error::WrongSignatureLength) => { + /// // The signature is not 144 bytes length + /// } + /// Err(_) => unreachable!() + /// } + /// ``` + /// + /// # Errors + /// + /// * [`Ed448Error::InvalidSignature`] if the signature is not valid, either the public key + /// or the signature used are not the right, or the message has been altered. + /// * [`Ed448Error::ContextTooLong`] if the optional context is more than 255 byte length. + /// * [`Ed448Error::WrongSignatureLength`] if the signature is not `SIG_LENGTH` byte. + #[inline] + pub fn verify(&self, msg: &[u8], sign: &[u8], ctx: Option<&[u8]>) -> crate::Result<()> { + self.verify_real(msg, sign, ctx, PreHash::False) + } + + /// Verify signature with public key. Message is pre-hashed before checked. + /// + /// See [`PublicKey::verify`] for more information. + /// + /// # Errors + /// + /// * [`Ed448Error::InvalidSignature`] if the signature is not valid, either the public key + /// or the signature used are not the right, or the message has been altered. + /// * [`Ed448Error::ContextTooLong`] if the optional context is more than 255 byte length. + /// * [`Ed448Error::WrongSignatureLength`] if the signature is not `SIG_LENGTH` byte. + #[inline] + pub fn verify_ph(&self, msg: &[u8], sign: &[u8], ctx: Option<&[u8]>) -> crate::Result<()> { + self.verify_real(msg, sign, ctx, PreHash::True) + } + + fn verify_real( + &self, + msg: &[u8], + sign: &[u8], + ctx: Option<&[u8]>, + pre_hash: PreHash, + ) -> crate::Result<()> { + // Sanity-check sizes. + if sign.len() < SIG_LENGTH { + return Err(Ed448Error::WrongSignatureLength); + } + + // Split signature into R and S, and parse. + let (Rraw, Sraw) = sign.split_at(KEY_LENGTH); + let (R, S) = ( + Point::decode(Rraw).map_err(|_| Ed448Error::InvalidSignature)?, + BigInt::from_bytes_le(Sign::Plus, Sraw), + ); + // Parse public key. + let A = Point::decode(&self.as_byte()).map_err(|_| Ed448Error::InvalidSignature)?; + if &S >= Point::l() { + return Err(Ed448Error::InvalidSignature); + } + // Calculate h. + let h = { + let (ctx, msg) = init_sig(ctx, pre_hash, msg)?; + shake256(vec![Rraw, &self.as_byte(), &msg], ctx.as_ref(), pre_hash) + }; + let h = BigInt::from_bytes_le(Sign::Plus, &h) % Point::l(); + // Calculate left and right sides of check eq. + let rhs = R + (A * h); + let lhs = Point::default() * S; + // Check eq. holds? + if lhs.double().double() == rhs.double().double() { + Ok(()) + } else { + Err(Ed448Error::InvalidSignature) + } + } +} + +/// Instantiate a `PublicKey` from the `PrivateKey`. +impl From<&PrivateKey> for PublicKey { + #[inline] + fn from(private_key: &PrivateKey) -> Self { + let (s, _) = &private_key.expand(); + // 3. Interpret the buffer as the little-endian integer, forming a + // secret scalar s. + Self::from(BigInt::from_bytes_le(Sign::Plus, s)) + } +} + +/// Do not use, it's for internal use only to generate the PublicKey +#[doc(hidden)] +impl From for PublicKey { + #[inline] + fn from(s: BigInt) -> Self { + // Perform a known-base-point scalar multiplication [s]B. + let A = Point::default() * s; + + // 4. The public key A is the encoding of the point [s]B. + Self(A) + } +} + +impl From<[u8; KEY_LENGTH]> for PublicKey { + #[inline] + fn from(array: [u8; KEY_LENGTH]) -> Self { + Self(Point::decode(&array).expect("bad ed448 public key")) + } +} + +impl From<&'_ [u8; KEY_LENGTH]> for PublicKey { + #[inline] + fn from(array: &'_ [u8; KEY_LENGTH]) -> Self { + Self(Point::decode(array).expect("bad ed448 public key")) + } +} + +impl TryFrom<&[u8]> for PublicKey { + type Error = Ed448Error; + + #[inline] + fn try_from(array: &[u8]) -> Result { + if array.len() != KEY_LENGTH { + return Err(Ed448Error::WrongPublicKeyLength); + } + Ok(Self(Point::decode(array)?)) + } +} + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::*; + use rand_core::OsRng; + + #[test] + fn test_vectors_rfc8032_public() { + let secret_vec = hex::decode( + "6c82a562cb808d10d632be89c8513ebf6c929f34ddfa8c9f63c9960ef6e348a3\ + 528c8a3fcc2f044e39a3fc5b94492f8f032e7549a20098f95b", + ) + .unwrap(); + let ref_public = hex::decode( + "5fd7449b59b461fd2ce787ec616ad46a1da1342485a70e1f8a0ea75d80e96778\ + edf124769b46c7061bd6783df1e50f6cd1fa1abeafe8256180", + ) + .unwrap(); + + let secret = PrivateKey::try_from(&secret_vec[..]).unwrap(); + let public = PublicKey::from(&secret); + + assert_eq!(&public.as_byte()[..], &ref_public[..]); + } + + #[test] + fn wrong_verification_with_another_pub_key() { + let secret_1 = PrivateKey::new(&mut OsRng); + let msg = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec nec."; + let sig_1 = secret_1.sign(msg, None).unwrap(); + let public_2 = PublicKey::from(&PrivateKey::new(&mut OsRng)); + assert_eq!( + public_2.verify(msg, &sig_1, None).unwrap_err(), + Ed448Error::InvalidSignature + ); + } + + #[test] + fn wrong_pubkey_length() { + let pub_key = PublicKey::try_from(&[0x01_u8][..]); + assert_eq!(pub_key.unwrap_err(), Ed448Error::WrongPublicKeyLength); + } + + #[test] + fn wrong_sign_length() { + let pubkey = PublicKey::from(&PrivateKey::new(&mut OsRng)); + let sig = [0x01; SIG_LENGTH - 1]; + assert_eq!( + pubkey.verify(b"message", &sig, None).unwrap_err(), + Ed448Error::WrongSignatureLength + ); + } + + #[test] + fn instantiate_pubkey() { + let pkey = PrivateKey::new(&mut OsRng); + let pkey_slice = *pkey.as_bytes(); + let pub_key1 = PublicKey::from(&pkey_slice); + let pub_key2 = PublicKey::from(pkey_slice); + + assert_eq!(pub_key1.as_byte(), pub_key2.as_byte()); + } + + #[test] + fn wrong_with_altered_message() { + let secret = PrivateKey::new(&mut OsRng); + let public = PublicKey::from(&PrivateKey::new(&mut OsRng)); + let msg_1 = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec nec."; + // One dot missing at the end + let msg_2 = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec nec"; + let sig = secret.sign(msg_1, None).unwrap(); + assert_eq!( + public.verify(msg_2, &sig, None).unwrap_err(), + Ed448Error::InvalidSignature + ); + } + + #[test] + fn wrong_with_forged_pub_key() { + let secret = PrivateKey::new(&mut OsRng); + let public = PublicKey::from(&[255; KEY_LENGTH]); + let msg = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec nec."; + // One dot missing at the end + let sig = secret.sign(msg, None).unwrap(); + assert_eq!( + public.verify(msg, &sig, None).unwrap_err(), + Ed448Error::InvalidSignature + ); + } + + #[test] + fn wrong_with_forged_signature() { + let secret = PrivateKey::new(&mut OsRng); + let public = PublicKey::from(&secret); + let msg = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec nec."; + // One dot missing at the end + let sig = [1; SIG_LENGTH]; + assert_eq!( + public.verify(msg, &sig, None).unwrap_err(), + Ed448Error::InvalidSignature + ); + } +} diff --git a/crates/ed448-rust/tests/rfc8032.rs b/crates/ed448-rust/tests/rfc8032.rs new file mode 100644 index 0000000..2450c83 --- /dev/null +++ b/crates/ed448-rust/tests/rfc8032.rs @@ -0,0 +1,215 @@ +use std::convert::TryFrom; + +use ed448_rust::{PrivateKey, PublicKey}; + +#[test] +fn ed448() { + vec![ + ( + "-----Blank", + "6c82a562cb808d10d632be89c8513ebf6c929f34ddfa8c9f63c9960ef6e348a3528c8a3fcc2f044e39a3\ + fc5b94492f8f032e7549a20098f95b", + "5fd7449b59b461fd2ce787ec616ad46a1da1342485a70e1f8a0ea75d80e96778edf124769b46c7061bd6\ + 783df1e50f6cd1fa1abeafe8256180", + "", + None, + "533a37f6bbe457251f023c0d88f976ae2dfb504a843e34d2074fd823d41a591f2b233f034f628281f2fd\ + 7a22ddd47d7828c59bd0a21bfd3980ff0d2028d4b18a9df63e006c5d1c2d345b925d8dc00b4104852\ + db99ac5c7cdda8530a113a0f4dbb61149f05a7363268c71d95808ff2e652600", + ), + ( + "-----1 octet", + "c4eab05d357007c632f3dbb48489924d552b08fe0c353a0d4a1f00acda2c463afbea67c5e8d2877c5e3b\ + c397a659949ef8021e954e0a12274e", + "43ba28f430cdff456ae531545f7ecd0ac834a55d9358c0372bfa0c6c6798c0866aea01eb00742802b843\ + 8ea4cb82169c235160627b4c3a9480", + "03", + None, + "26b8f91727bd62897af15e41eb43c377efb9c610d48f2335cb0bd0087810f4352541b143c4b981b7e18f\ + 62de8ccdf633fc1bf037ab7cd779805e0dbcc0aae1cbcee1afb2e027df36bc04dcecbf154336c19f0\ + af7e0a6472905e799f1953d2a0ff3348ab21aa4adafd1d234441cf807c03a00", + ), + ( + "-----1 octet (with context)", + "c4eab05d357007c632f3dbb48489924d552b08fe0c353a0d4a1f00acda2c463afbea67c5e8d2877c5e3b\ + c397a659949ef8021e954e0a12274e", + "43ba28f430cdff456ae531545f7ecd0ac834a55d9358c0372bfa0c6c6798c0866aea01eb00742802b843\ + 8ea4cb82169c235160627b4c3a9480", + "03", + Some("666f6f"), + "d4f8f6131770dd46f40867d6fd5d5055de43541f8c5e35abbcd001b32a89f7d2151f7647f11d8ca2ae27\ + 9fb842d607217fce6e042f6815ea000c85741de5c8da1144a6a1aba7f96de42505d7a7298524fda53\ + 8fccbbb754f578c1cad10d54d0d5428407e85dcbc98a49155c13764e66c3c00", + ), + ( + "-----11 octets", + "cd23d24f714274e744343237b93290f511f6425f98e64459ff203e8985083ffdf60500553abc0e05cd02\ + 184bdb89c4ccd67e187951267eb328", + "dcea9e78f35a1bf3499a831b10b86c90aac01cd84b67a0109b55a36e9328b1e365fce161d71ce7131a54\ + 3ea4cb5f7e9f1d8b00696447001400", + "0c3e544074ec63b0265e0c", + None, + "1f0a8888ce25e8d458a21130879b840a9089d999aaba039eaf3e3afa090a09d389dba82c4ff2ae8ac5cd\ + fb7c55e94d5d961a29fe0109941e00b8dbdeea6d3b051068df7254c0cdc129cbe62db2dc957dbb47b\ + 51fd3f213fb8698f064774250a5028961c9bf8ffd973fe5d5c206492b140e00", + ), + ( + "-----12 octets", + "258cdd4ada32ed9c9ff54e63756ae582fb8fab2ac721f2c8e676a72768513d939f63dddb55609133f29a\ + df86ec9929dccb52c1c5fd2ff7e21b", + "3ba16da0c6f2cc1f30187740756f5e798d6bc5fc015d7c63cc9510ee3fd44adc24d8e968b6e46e6f94d1\ + 9b945361726bd75e149ef09817f580", + "64a65f3cdedcdd66811e2915", + None, + "7eeeab7c4e50fb799b418ee5e3197ff6bf15d43a14c34389b59dd1a7b1b85b4ae90438aca634bea45e3a\ + 2695f1270f07fdcdf7c62b8efeaf00b45c2c96ba457eb1a8bf075a3db28e5c24f6b923ed4ad747c3c\ + 9e03c7079efb87cb110d3a99861e72003cbae6d6b8b827e4e6c143064ff3c00", + ), + ( + "-----13 octets", + "7ef4e84544236752fbb56b8f31a23a10e42814f5f55ca037cdcc11c64c9a3b2949c1bb60700314611732\ + a6c2fea98eebc0266a11a93970100e", + "b3da079b0aa493a5772029f0467baebee5a8112d9d3a22532361da294f7bb3815c5dc59e176b4d9f381c\ + a0938e13c6c07b174be65dfa578e80", + "64a65f3cdedcdd66811e2915e7", + None, + "6a12066f55331b6c22acd5d5bfc5d71228fbda80ae8dec26bdd306743c5027cb4890810c162c02746867\ + 5ecf645a83176c0d7323a2ccde2d80efe5a1268e8aca1d6fbc194d3f77c44986eb4ab4177919ad8be\ + c33eb47bbb5fc6e28196fd1caf56b4e7e0ba5519234d047155ac727a1053100", + ), + ( + "-----64 octets", + "d65df341ad13e008567688baedda8e9dcdc17dc024974ea5b4227b6530e339bff21f99e68ca6968f3cca\ + 6dfe0fb9f4fab4fa135d5542ea3f01", + "df9705f58edbab802c7f8363cfe5560ab1c6132c20a9f1dd163483a26f8ac53a39d6808bf4a1dfbd261b\ + 099bb03b3fb50906cb28bd8a081f00", + "bd0f6a3747cd561bdddf4640a332461a4a30a12a434cd0bf40d766d9c6d458e5512204a30c17d1f50b50\ + 79631f64eb3112182da3005835461113718d1a5ef944", + None, + "554bc2480860b49eab8532d2a533b7d578ef473eeb58c98bb2d0e1ce488a98b18dfde9b9b90775e67f47\ + d4a1c3482058efc9f40d2ca033a0801b63d45b3b722ef552bad3b4ccb667da350192b61c508cf7b6b\ + 5adadc2c8d9a446ef003fb05cba5f30e88e36ec2703b349ca229c2670833900", + ), + ( + "-----256 octets", + "2ec5fe3c17045abdb136a5e6a913e32ab75ae68b53d2fc149b77e504132d37569b7e766ba74a19bd6162\ + 343a21c8590aa9cebca9014c636df5", + "79756f014dcfe2079f5dd9e718be4171e2ef2486a08f25186f6bff43a9936b9bfe12402b08ae65798a3d\ + 81e22e9ec80e7690862ef3d4ed3a00", + "15777532b0bdd0d1389f636c5f6b9ba734c90af572877e2d272dd078aa1e567cfa80e12928bb542330e8\ + 409f3174504107ecd5efac61ae7504dabe2a602ede89e5cca6257a7c77e27a702b3ae39fc769fc54f\ + 2395ae6a1178cab4738e543072fc1c177fe71e92e25bf03e4ecb72f47b64d0465aaea4c7fad372536\ + c8ba516a6039c3c2a39f0e4d832be432dfa9a706a6e5c7e19f397964ca4258002f7c0541b590316db\ + c5622b6b2a6fe7a4abffd96105eca76ea7b98816af0748c10df048ce012d901015a51f189f3888145\ + c03650aa23ce894c3bd889e030d565071c59f409a9981b51878fd6fc110624dcbcde0bf7a69ccce38\ + fabdf86f3bef6044819de11", + None, + "c650ddbb0601c19ca11439e1640dd931f43c518ea5bea70d3dcde5f4191fe53f00cf966546b72bcc7d58\ + be2b9badef28743954e3a44a23f880e8d4f1cfce2d7a61452d26da05896f0a50da66a239a8a188b6d\ + 825b3305ad77b73fbac0836ecc60987fd08527c1a8e80d5823e65cafe2a3d00", + ), + ( + "-----1023 octets", + "872d093780f5d3730df7c212664b37b8a0f24f56810daa8382cd4fa3f77634ec44dc54f1c2ed9bea86fa\ + fb7632d8be199ea165f5ad55dd9ce8", + "a81b2e8a70a5ac94ffdbcc9badfc3feb0801f258578bb114ad44ece1ec0e799da08effb81c5d685c0c56\ + f64eecaef8cdf11cc38737838cf400", + "6ddf802e1aae4986935f7f981ba3f0351d6273c0a0c22c9c0e8339168e675412a3debfaf435ed6515580\ + 07db4384b650fcc07e3b586a27a4f7a00ac8a6fec2cd86ae4bf1570c41e6a40c931db27b2faa15a8c\ + edd52cff7362c4e6e23daec0fbc3a79b6806e316efcc7b68119bf46bc76a26067a53f296dafdbdc11\ + c77f7777e972660cf4b6a9b369a6665f02e0cc9b6edfad136b4fabe723d2813db3136cfde9b6d0443\ + 22fee2947952e031b73ab5c603349b307bdc27bc6cb8b8bbd7bd323219b8033a581b59eadebb09b3c\ + 4f3d2277d4f0343624acc817804728b25ab797172b4c5c21a22f9c7839d64300232eb66e53f31c723\ + fa37fe387c7d3e50bdf9813a30e5bb12cf4cd930c40cfb4e1fc622592a49588794494d56d24ea4b40\ + c89fc0596cc9ebb961c8cb10adde976a5d602b1c3f85b9b9a001ed3c6a4d3b1437f52096cd1956d04\ + 2a597d561a596ecd3d1735a8d570ea0ec27225a2c4aaff26306d1526c1af3ca6d9cf5a2c98f47e1c4\ + 6db9a33234cfd4d81f2c98538a09ebe76998d0d8fd25997c7d255c6d66ece6fa56f11144950f02779\ + 5e653008f4bd7ca2dee85d8e90f3dc315130ce2a00375a318c7c3d97be2c8ce5b6db41a6254ff264f\ + a6155baee3b0773c0f497c573f19bb4f4240281f0b1f4f7be857a4e59d416c06b4c50fa09e1810ddc\ + 6b1467baeac5a3668d11b6ecaa901440016f389f80acc4db977025e7f5924388c7e340a732e554440\ + e76570f8dd71b7d640b3450d1fd5f0410a18f9a3494f707c717b79b4bf75c98400b096b21653b5d21\ + 7cf3565c9597456f70703497a078763829bc01bb1cbc8fa04eadc9a6e3f6699587a9e75c94e5bab00\ + 36e0b2e711392cff0047d0d6b05bd2a588bc109718954259f1d86678a579a3120f19cfb2963f177ae\ + b70f2d4844826262e51b80271272068ef5b3856fa8535aa2a88b2d41f2a0e2fda7624c2850272ac4a\ + 2f561f8f2f7a318bfd5caf9696149e4ac824ad3460538fdc25421beec2cc6818162d06bbed0c40a38\ + 7192349db67a118bada6cd5ab0140ee273204f628aad1c135f770279a651e24d8c14d75a6059d76b9\ + 6a6fd857def5e0b354b27ab937a5815d16b5fae407ff18222c6d1ed263be68c95f32d908bd895cd76\ + 207ae726487567f9a67dad79abec316f683b17f2d02bf07e0ac8b5bc6162cf94697b3c27cd1fea49b\ + 27f23ba2901871962506520c392da8b6ad0d99f7013fbc06c2c17a569500c8a7696481c1cd33e9b14\ + e40b82e79a5f5db82571ba97bae3ad3e0479515bb0e2b0f3bfcd1fd33034efc6245eddd7ee2086dda\ + e2600d8ca73e214e8c2b0bdb2b047c6a464a562ed77b73d2d841c4b34973551257713b753632efba3\ + 48169abc90a68f42611a40126d7cb21b58695568186f7e569d2ff0f9e745d0487dd2eb997cafc5abf\ + 9dd102e62ff66cba87", + None, + "e301345a41a39a4d72fff8df69c98075a0cc082b802fc9b2b6bc503f926b65bddf7f4c8f1cb49f6396af\ + c8a70abe6d8aef0db478d4c6b2970076c6a0484fe76d76b3a97625d79f1ce240e7c576750d2955282\ + 86f719b413de9ada3e8eb78ed573603ce30d8bb761785dc30dbc320869e1a00", + ), + ] + .iter() + .for_each(|(name, pkey, pub_key, msg, context, sig)| { + println!("Test: {}", name); + let pkey = hex::decode(pkey).unwrap(); + let pub_key = hex::decode(pub_key).unwrap(); + let msg = hex::decode(msg).unwrap(); + let context = context.map(|ctx| hex::decode(ctx).unwrap()); + let sig = hex::decode(sig).unwrap(); + + let pkey = PrivateKey::try_from(&pkey[..]).unwrap(); + let pub_k = PublicKey::from(&pkey); + assert_eq!(&pub_k.as_byte()[..], &pub_key[..]); + + let sig_ = pkey.sign(&msg, context.as_deref()).unwrap(); + assert_eq!(&sig_[..], &sig[..]); + + pub_k.verify(&msg, &sig, context.as_deref()).unwrap(); + }) +} + +#[test] +fn ed448ph() { + vec![ + ( + "-----TEST abc", + "833fe62409237b9d62ec77587520911e9a759cec1d19755b7da901b96dca3d42ef7822e0d5104127dc05\ + d6dbefde69e3ab2cec7c867c6e2c49", + "259b71c19f83ef77a7abd26524cbdb3161b590a48f7d17de3ee0ba9c52beb743c09428a131d6b1b57303\ + d90d8132c276d5ed3d5d01c0f53880", + "616263", + None, + "822f6901f7480f3d5f562c592994d9693602875614483256505600bbc281ae381f54d6bce2ea91157493\ + 2f52a4e6cadd78769375ec3ffd1b801a0d9b3f4030cd433964b6457ea39476511214f97469b57dd32\ + dbc560a9a94d00bff07620464a3ad203df7dc7ce360c3cd3696d9d9fab90f00", + ), + ( + "-----TEST abc (with context)", + "833fe62409237b9d62ec77587520911e9a759cec1d19755b7da901b96dca3d42ef7822e0d5104127dc05\ + d6dbefde69e3ab2cec7c867c6e2c49", + "259b71c19f83ef77a7abd26524cbdb3161b590a48f7d17de3ee0ba9c52beb743c09428a131d6b1b57303\ + d90d8132c276d5ed3d5d01c0f53880", + "616263", + Some("666f6f"), + "c32299d46ec8ff02b54540982814dce9a05812f81962b649d528095916a2aa481065b1580423ef927ecf\ + 0af5888f90da0f6a9a85ad5dc3f280d91224ba9911a3653d00e484e2ce232521481c8658df304bb77\ + 45a73514cdb9bf3e15784ab71284f8d0704a608c54a6b62d97beb511d132100", + ), + ] + .iter() + .for_each(|(name, pkey, pub_key, msg, context, sig)| { + println!("Test pre-hashed: {}", name); + let pkey = hex::decode(pkey).unwrap(); + let pub_key = hex::decode(pub_key).unwrap(); + let msg = hex::decode(msg).unwrap(); + let context = context.map(|ctx| hex::decode(ctx).unwrap()); + let sig = hex::decode(sig).unwrap(); + + let pkey = PrivateKey::try_from(&pkey[..]).unwrap(); + let pub_k = PublicKey::from(&pkey); + assert_eq!(&pub_k.as_byte()[..], &pub_key[..]); + + let sig_ = pkey.sign_ph(&msg, context.as_deref()).unwrap(); + assert_eq!(&sig_[..], &sig[..]); + + pub_k.verify_ph(&msg, &sig, context.as_deref()).unwrap(); + }) +} diff --git a/node/config/version.go b/node/config/version.go index 4d4b762..c7ed605 100644 --- a/node/config/version.go +++ b/node/config/version.go @@ -43,9 +43,9 @@ func FormatVersion(version []byte) string { } func GetPatchNumber() byte { - return 0x03 + return 0x00 } func GetRCNumber() byte { - return 0x00 + return 0x03 } diff --git a/node/consensus/data/data_clock_consensus_engine.go b/node/consensus/data/data_clock_consensus_engine.go index 3216c4f..5175f1d 100644 --- a/node/consensus/data/data_clock_consensus_engine.go +++ b/node/consensus/data/data_clock_consensus_engine.go @@ -566,8 +566,6 @@ func (e *DataClockConsensusEngine) Start() <-chan error { errChan <- nil }() - go e.runPreMidnightProofWorker() - e.wg.Add(1) go func() { defer e.wg.Done() diff --git a/node/consensus/data/peer_messaging.go b/node/consensus/data/peer_messaging.go index 8433415..9c0afd9 100644 --- a/node/consensus/data/peer_messaging.go +++ b/node/consensus/data/peer_messaging.go @@ -666,3 +666,25 @@ func (e *DataClockConsensusEngine) GetPublicChannel( ) error { return errors.New("not supported") } + +func GetAddressOfPreCoinProof( + proof *protobufs.PreCoinProof, +) ([]byte, error) { + eval := []byte{} + eval = append(eval, application.TOKEN_ADDRESS...) + eval = append(eval, proof.Amount...) + eval = binary.BigEndian.AppendUint32(eval, proof.Index) + eval = append(eval, proof.IndexProof...) + eval = append(eval, proof.Commitment...) + eval = append(eval, proof.Proof...) + eval = binary.BigEndian.AppendUint32(eval, proof.Parallelism) + eval = binary.BigEndian.AppendUint32(eval, proof.Difficulty) + eval = binary.BigEndian.AppendUint32(eval, 0) + eval = append(eval, proof.Owner.GetImplicitAccount().Address...) + addressBI, err := poseidon.HashBytes(eval) + if err != nil { + return nil, err + } + + return addressBI.FillBytes(make([]byte, 32)), nil +} diff --git a/node/consensus/data/pre_midnight_proof_worker.go b/node/consensus/data/pre_midnight_proof_worker.go deleted file mode 100644 index 5eab95e..0000000 --- a/node/consensus/data/pre_midnight_proof_worker.go +++ /dev/null @@ -1,300 +0,0 @@ -package data - -import ( - "bytes" - "encoding/binary" - "strings" - "time" - - "github.com/iden3/go-iden3-crypto/poseidon" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/pkg/errors" - "go.uber.org/zap" - "google.golang.org/grpc" - "source.quilibrium.com/quilibrium/monorepo/node/config" - "source.quilibrium.com/quilibrium/monorepo/node/consensus" - "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" - "source.quilibrium.com/quilibrium/monorepo/node/protobufs" - "source.quilibrium.com/quilibrium/monorepo/node/store" -) - -func (e *DataClockConsensusEngine) runPreMidnightProofWorker() { - e.logger.Info("checking for pre-2.0 proofs") - - increment, _, _, err := e.dataProofStore.GetLatestDataTimeProof( - e.pubSub.GetPeerID(), - ) - if err != nil { - if errors.Is(err, store.ErrNotFound) { - e.logger.Info("could not find pre-2.0 proofs") - return - } - - panic(err) - } - - for { - if e.GetState() < consensus.EngineStateCollecting { - e.logger.Info("waiting for node to finish starting") - time.Sleep(10 * time.Second) - continue - } - break - } - - addrBI, err := poseidon.HashBytes(e.pubSub.GetPeerID()) - if err != nil { - panic(err) - } - - addr := addrBI.FillBytes(make([]byte, 32)) - - genesis := config.GetGenesis() - pub, err := crypto.UnmarshalEd448PublicKey(genesis.Beacon) - if err != nil { - panic(err) - } - - peerId, err := peer.IDFromPublicKey(pub) - if err != nil { - panic(errors.Wrap(err, "error getting peer id")) - } - - for { - tries := e.GetFrameProverTries() - - if len(tries) == 0 || e.pubSub.GetNetworkPeersCount() < 3 { - e.logger.Info("waiting for more peer info to appear") - time.Sleep(10 * time.Second) - continue - } - - _, prfs, err := e.coinStore.GetPreCoinProofsForOwner(addr) - if err != nil && !errors.Is(err, store.ErrNotFound) { - e.logger.Error("error while fetching pre-coin proofs", zap.Error(err)) - return - } - - if len(prfs) != 0 { - e.logger.Info("already completed pre-midnight mint") - return - } - - break - } - - resume := make([]byte, 32) - cc, err := e.pubSub.GetDirectChannel(e.ctx, []byte(peerId), "worker") - if err != nil { - e.logger.Info( - "could not establish direct channel, waiting...", - zap.Error(err), - ) - time.Sleep(10 * time.Second) - } - for { - state := e.GetState() - if state >= consensus.EngineStateStopping || state == consensus.EngineStateStopped { - break - } - _, prfs, err := e.coinStore.GetPreCoinProofsForOwner(addr) - if err != nil && !errors.Is(err, store.ErrNotFound) { - e.logger.Error("error while fetching pre-coin proofs", zap.Error(err)) - return - } - - if len(prfs) != 0 { - e.logger.Info("already completed pre-midnight mint") - return - } - - if cc == nil { - cc, err = e.pubSub.GetDirectChannel(e.ctx, []byte(peerId), "worker") - if err != nil { - e.logger.Info( - "could not establish direct channel, waiting...", - zap.Error(err), - ) - cc = nil - time.Sleep(10 * time.Second) - continue - } - } - - client := protobufs.NewDataServiceClient(cc) - - if bytes.Equal(resume, make([]byte, 32)) { - status, err := client.GetPreMidnightMintStatus( - e.ctx, - &protobufs.PreMidnightMintStatusRequest{ - Owner: addr, - }, - grpc.MaxCallSendMsgSize(1*1024*1024), - grpc.MaxCallRecvMsgSize(1*1024*1024), - ) - if err != nil || status == nil { - e.logger.Error( - "got error response, waiting...", - zap.Error(err), - ) - time.Sleep(10 * time.Second) - cc.Close() - cc = nil - err = e.pubSub.Reconnect([]byte(peerId)) - if err != nil { - e.logger.Error( - "got error response, waiting...", - zap.Error(err), - ) - time.Sleep(10 * time.Second) - } - continue - } - - resume = status.Address - - if status.Increment != 0 { - increment = status.Increment - 1 - } else if !bytes.Equal(status.Address, make([]byte, 32)) { - increment = 0 - } - } - - proofs := [][]byte{ - []byte("pre-dusk"), - resume, - } - - batchCount := 0 - // the cast is important, it underflows without: - for i := int(increment); i >= 0; i-- { - _, parallelism, input, output, err := e.dataProofStore.GetDataTimeProof( - e.pubSub.GetPeerID(), - uint32(i), - ) - if err == nil { - p := []byte{} - p = binary.BigEndian.AppendUint32(p, uint32(i)) - p = binary.BigEndian.AppendUint32(p, parallelism) - p = binary.BigEndian.AppendUint64(p, uint64(len(input))) - p = append(p, input...) - p = binary.BigEndian.AppendUint64(p, uint64(len(output))) - p = append(p, output...) - - proofs = append(proofs, p) - } else { - e.logger.Error( - "could not find data time proof for peer and increment, stopping worker", - zap.String("peer_id", peer.ID(e.pubSub.GetPeerID()).String()), - zap.Int("increment", i), - ) - cc.Close() - cc = nil - return - } - - batchCount++ - if batchCount == 200 || i == 0 { - e.logger.Info("publishing proof batch", zap.Int("increment", i)) - - payload := []byte("mint") - for _, i := range proofs { - payload = append(payload, i...) - } - sig, err := e.pubSub.SignMessage(payload) - if err != nil { - cc.Close() - panic(err) - } - - resp, err := client.HandlePreMidnightMint( - e.ctx, - &protobufs.MintCoinRequest{ - Proofs: proofs, - Signature: &protobufs.Ed448Signature{ - PublicKey: &protobufs.Ed448PublicKey{ - KeyValue: e.pubSub.GetPublicKey(), - }, - Signature: sig, - }, - }, - grpc.MaxCallSendMsgSize(1*1024*1024), - grpc.MaxCallRecvMsgSize(1*1024*1024), - ) - - if err != nil { - if strings.Contains( - err.Error(), - application.ErrInvalidStateTransition.Error(), - ) && i == 0 { - resume = make([]byte, 32) - e.logger.Info("pre-midnight proofs submitted, returning") - cc.Close() - cc = nil - return - } - - e.logger.Error( - "got error response, waiting...", - zap.Error(err), - ) - - resume = make([]byte, 32) - cc.Close() - cc = nil - time.Sleep(10 * time.Second) - err = e.pubSub.Reconnect([]byte(peerId)) - if err != nil { - e.logger.Error( - "got error response, waiting...", - zap.Error(err), - ) - time.Sleep(10 * time.Second) - } - break - } - - resume = resp.Address - batchCount = 0 - proofs = [][]byte{ - []byte("pre-dusk"), - resume, - } - - if i == 0 { - e.logger.Info("pre-midnight proofs submitted, returning") - cc.Close() - cc = nil - return - } else { - increment = uint32(i) - 1 - } - - break - } - } - } -} - -func GetAddressOfPreCoinProof( - proof *protobufs.PreCoinProof, -) ([]byte, error) { - eval := []byte{} - eval = append(eval, application.TOKEN_ADDRESS...) - eval = append(eval, proof.Amount...) - eval = binary.BigEndian.AppendUint32(eval, proof.Index) - eval = append(eval, proof.IndexProof...) - eval = append(eval, proof.Commitment...) - eval = append(eval, proof.Proof...) - eval = binary.BigEndian.AppendUint32(eval, proof.Parallelism) - eval = binary.BigEndian.AppendUint32(eval, proof.Difficulty) - eval = binary.BigEndian.AppendUint32(eval, 0) - eval = append(eval, proof.Owner.GetImplicitAccount().Address...) - addressBI, err := poseidon.HashBytes(eval) - if err != nil { - return nil, err - } - - return addressBI.FillBytes(make([]byte, 32)), nil -} diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index c1911f6..73d50f4 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -17,8 +17,8 @@ func init() { } const ( - BranchNodes = 1024 - BranchBits = 10 // log2(1024) + BranchNodes = 64 + BranchBits = 6 // log2(64) BranchMask = BranchNodes - 1 ) @@ -73,7 +73,7 @@ func (n *VectorCommitmentBranchNode) Commit() []byte { } } - n.Commitment = rbls48581.CommitRaw(data, 1024) + n.Commitment = rbls48581.CommitRaw(data, 64) } return n.Commitment @@ -103,7 +103,7 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool { } } - n.Commitment = rbls48581.CommitRaw(data, 1024) + n.Commitment = rbls48581.CommitRaw(data, 64) data = data[64*index : 64*(index+1)] } else { child := n.Children[index] @@ -127,7 +127,7 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool { } } - return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 1024) + return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 64) } func (n *VectorCommitmentBranchNode) Prove(index int) []byte { @@ -153,7 +153,7 @@ func (n *VectorCommitmentBranchNode) Prove(index int) []byte { } } - return rbls48581.ProveRaw(data, uint64(index), 1024) + return rbls48581.ProveRaw(data, uint64(index), 64) } type VectorCommitmentTree struct { diff --git a/node/crypto/proof_tree_test.go b/node/crypto/proof_tree_test.go index cf1d3a9..c7f186b 100644 --- a/node/crypto/proof_tree_test.go +++ b/node/crypto/proof_tree_test.go @@ -9,6 +9,72 @@ import ( "source.quilibrium.com/quilibrium/monorepo/bls48581/generated/bls48581" ) +func BenchmarkVectorCommitmentTreeInsert(b *testing.B) { + tree := &VectorCommitmentTree{} + addresses := [][]byte{} + + for i := range b.N { + d := make([]byte, 32) + rand.Read(d) + addresses = append(addresses, d) + err := tree.Insert(d, d) + if err != nil { + b.Errorf("Failed to insert item %d: %v", i, err) + } + } +} + +func BenchmarkVectorCommitmentTreeCommit(b *testing.B) { + tree := &VectorCommitmentTree{} + addresses := [][]byte{} + + for i := range b.N { + d := make([]byte, 32) + rand.Read(d) + addresses = append(addresses, d) + err := tree.Insert(d, d) + if err != nil { + b.Errorf("Failed to insert item %d: %v", i, err) + } + tree.Commit() + } +} + +func BenchmarkVectorCommitmentTreeProve(b *testing.B) { + tree := &VectorCommitmentTree{} + addresses := [][]byte{} + + for i := range b.N { + d := make([]byte, 32) + rand.Read(d) + addresses = append(addresses, d) + err := tree.Insert(d, d) + if err != nil { + b.Errorf("Failed to insert item %d: %v", i, err) + } + tree.Prove(d) + } +} + +func BenchmarkVectorCommitmentTreeVerify(b *testing.B) { + tree := &VectorCommitmentTree{} + addresses := [][]byte{} + + for i := range b.N { + d := make([]byte, 32) + rand.Read(d) + addresses = append(addresses, d) + err := tree.Insert(d, d) + if err != nil { + b.Errorf("Failed to insert item %d: %v", i, err) + } + p := tree.Prove(d) + if !tree.Verify(d, p) { + b.Errorf("bad proof") + } + } +} + func TestVectorCommitmentTrees(t *testing.T) { bls48581.Init() tree := &VectorCommitmentTree{} @@ -147,23 +213,23 @@ func TestVectorCommitmentTrees(t *testing.T) { tree = &VectorCommitmentTree{} - // Empty tree should have zero hash - emptyRoot := tree.Root() - if len(emptyRoot) != 64 { - t.Errorf("Expected 64 byte root hash, got %d bytes", len(emptyRoot)) + // Empty tree should be empty + emptyRoot := tree.Root + if emptyRoot != nil { + t.Errorf("Expected empty root") } // Root should change after insert tree.Insert([]byte("key1"), []byte("value1")) - firstRoot := tree.Root() + firstRoot := tree.Root.Commit() - if bytes.Equal(firstRoot, emptyRoot) { + if bytes.Equal(firstRoot, bytes.Repeat([]byte{0x00}, 64)) { t.Error("Root hash should change after insert") } // Root should change after update tree.Insert([]byte("key1"), []byte("value2")) - secondRoot := tree.Root() + secondRoot := tree.Root.Commit() if bytes.Equal(secondRoot, firstRoot) { t.Error("Root hash should change after update") @@ -171,13 +237,14 @@ func TestVectorCommitmentTrees(t *testing.T) { // Root should change after delete tree.Delete([]byte("key1")) - thirdRoot := tree.Root() + thirdRoot := tree.Root - if !bytes.Equal(thirdRoot, emptyRoot) { + if thirdRoot != nil { t.Error("Root hash should match empty tree after deleting all entries") } tree = &VectorCommitmentTree{} + cmptree := &VectorCommitmentTree{} addresses := [][]byte{} @@ -187,7 +254,7 @@ func TestVectorCommitmentTrees(t *testing.T) { addresses = append(addresses, d) } - // Insert 100 items + // Insert 1000 items for i := 0; i < 1000; i++ { key := addresses[i] value := addresses[i] @@ -197,6 +264,16 @@ func TestVectorCommitmentTrees(t *testing.T) { } } + // Insert 1000 items in reverse + for i := 999; i >= 0; i-- { + key := addresses[i] + value := addresses[i] + err := cmptree.Insert(key, value) + if err != nil { + t.Errorf("Failed to insert item %d: %v", i, err) + } + } + // Verify all items for i := 0; i < 1000; i++ { key := addresses[i] @@ -205,9 +282,23 @@ func TestVectorCommitmentTrees(t *testing.T) { if err != nil { t.Errorf("Failed to get item %d: %v", i, err) } + cmpvalue, err := cmptree.Get(key) + if err != nil { + t.Errorf("Failed to get item %d: %v", i, err) + } if !bytes.Equal(value, expected) { t.Errorf("Item %d: expected %x, got %x", i, string(expected), string(value)) } + if !bytes.Equal(value, cmpvalue) { + t.Errorf("Item %d: expected %x, got %x", i, string(value), string(cmpvalue)) + } + } + + tcommit := tree.Root.Commit() + cmptcommit := cmptree.Root.Commit() + + if !bytes.Equal(tcommit, cmptcommit) { + t.Errorf("tree mismatch, %x, %x", tcommit, cmptcommit) } proofs := tree.Prove(addresses[500]) diff --git a/node/execution/intrinsics/token/application/token_handle_mint.go b/node/execution/intrinsics/token/application/token_handle_mint.go index 0bf2020..d97eb1e 100644 --- a/node/execution/intrinsics/token/application/token_handle_mint.go +++ b/node/execution/intrinsics/token/application/token_handle_mint.go @@ -160,12 +160,12 @@ func (a *TokenApplication) preProcessMint( ) } - // Current frame - 2 is because the current frame is the newly created frame, + // Current frame - 1 is because the current frame is the newly created frame, // and the provers are submitting proofs on the frame preceding the one they // last saw. This enforces liveness and creates a punishment for being // late. if (previousFrame != nil && newFrameNumber <= previousFrame.FrameNumber) || - newFrameNumber < currentFrameNumber-2 { + newFrameNumber < currentFrameNumber-1 { previousFrameNumber := uint64(0) if previousFrame != nil { previousFrameNumber = previousFrame.FrameNumber @@ -338,12 +338,12 @@ func (a *TokenApplication) preProcessMint( ) } - // Current frame - 2 is because the current frame is the newly created frame, + // Current frame - 1 is because the current frame is the newly created frame, // and the provers are submitting proofs on the frame preceding the one they // last saw. This enforces liveness and creates a punishment for being // late. if (previousFrame != nil && newFrameNumber <= previousFrame.FrameNumber) || - newFrameNumber < currentFrameNumber-2 { + newFrameNumber < currentFrameNumber-1 { previousFrameNumber := uint64(0) if previousFrame != nil { previousFrameNumber = previousFrame.FrameNumber diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 2e3ce6e..f0f35d9 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -5,6 +5,7 @@ import ( "context" "crypto" "encoding/hex" + "fmt" "math/big" "slices" "strconv" @@ -346,13 +347,21 @@ func NewTokenExecutionEngine( e.proverPublicKey = publicKeyBytes e.provingKeyAddress = provingKeyAddress - e.stateTree, err = e.clockStore.GetDataStateTree(e.intrinsicFilter) - if err != nil && !errors.Is(err, store.ErrNotFound) { - panic(err) - } - - if e.stateTree == nil { + frame, _, err := e.clockStore.GetLatestDataClockFrame(e.intrinsicFilter) + if err != nil || frame.FrameNumber < 184479 { e.rebuildStateTree() + } else { + e.stateTree, err = e.clockStore.GetDataStateTree(e.intrinsicFilter) + if err != nil && !errors.Is(err, store.ErrNotFound) { + e.logger.Error( + "error encountered while fetching state tree, rebuilding", + zap.Error(err), + ) + } + + if e.stateTree == nil { + e.rebuildStateTree() + } } e.wg.Add(1) @@ -446,6 +455,15 @@ func (e *TokenExecutionEngine) rebuildStateTree() { panic(err) } + e.logger.Info("committing state tree") + + root := e.stateTree.Commit() + + e.logger.Info( + "committed state tree", + zap.String("root", fmt.Sprintf("%x", root)), + ) + err = e.clockStore.SetDataStateTree(txn, e.intrinsicFilter, e.stateTree) if err != nil { txn.Abort() @@ -1029,6 +1047,15 @@ func (e *TokenExecutionEngine) ProcessFrame( } } + e.logger.Info("committing state tree") + + root := stateTree.Commit() + + e.logger.Info( + "commited state tree", + zap.String("root", fmt.Sprintf("%x", root)), + ) + err = e.clockStore.SetDataStateTree( txn, e.intrinsicFilter, diff --git a/node/hypergraph/application/hypergraph.go b/node/hypergraph/application/hypergraph.go index 2b38caf..5400ef1 100644 --- a/node/hypergraph/application/hypergraph.go +++ b/node/hypergraph/application/hypergraph.go @@ -1,7 +1,6 @@ package application import ( - "encoding/binary" "errors" "source.quilibrium.com/quilibrium/monorepo/node/p2p" @@ -15,32 +14,36 @@ var ErrInvalidLocation = errors.New("invalid location") var ErrMissingExtrinsics = errors.New("missing extrinsics") var ErrIsExtrinsic = errors.New("is extrinsic") +// Extract only needed methods of VEnc interface +type Encrypted interface { + ToBytes() []byte + Verify(proof []byte) bool +} + type Vertex struct { - AppAddress [32]byte - DataAddress [32]byte - SegmentOrder uint16 + AppAddress [32]byte + DataAddress [32]byte + Data Encrypted } type Hyperedge struct { AppAddress [32]byte DataAddress [32]byte - Index uint16 - Extrinsics map[[66]byte]Atom + Extrinsics map[[64]byte]Atom } type Atom interface { - GetID() [66]byte + GetID() [64]byte GetAtomType() AtomType GetLocation() Location GetAppAddress() [32]byte GetDataAddress() [32]byte } -func (v *Vertex) GetID() [66]byte { - id := [66]byte{} +func (v *Vertex) GetID() [64]byte { + id := [64]byte{} copy(id[:32], v.AppAddress[:]) copy(id[32:64], v.DataAddress[:]) - binary.BigEndian.PutUint16(id[64:], v.SegmentOrder) return id } @@ -63,11 +66,10 @@ func (v *Vertex) GetDataAddress() [32]byte { return v.DataAddress } -func (h *Hyperedge) GetID() [66]byte { - id := [66]byte{} +func (h *Hyperedge) GetID() [64]byte { + id := [64]byte{} copy(id[:32], h.AppAddress[:]) copy(id[32:], h.DataAddress[:]) - binary.BigEndian.PutUint16(id[64:], h.Index) return id } @@ -92,7 +94,7 @@ func (h *Hyperedge) GetDataAddress() [32]byte { type ShardAddress struct { L1 [3]byte - L2 [48]byte + L2 [64]byte } func GetShardAddress(a Atom) ShardAddress { @@ -101,17 +103,17 @@ func GetShardAddress(a Atom) ShardAddress { return ShardAddress{ L1: [3]byte(p2p.GetBloomFilterIndices(appAddress[:], 256, 3)), - L2: [48]byte(p2p.GetBloomFilterIndices(append(append([]byte{}, appAddress[:]...), dataAddress[:]...), 65536, 24)), + L2: [64]byte(append(append([]byte{}, appAddress[:]...), dataAddress[:]...)), } } type IdSet struct { atomType AtomType - atoms map[[66]byte]Atom + atoms map[[64]byte]Atom } func NewIdSet(atomType AtomType) *IdSet { - return &IdSet{atomType: atomType, atoms: make(map[[66]byte]Atom)} + return &IdSet{atomType: atomType, atoms: make(map[[64]byte]Atom)} } func (set *IdSet) Add(atom Atom) error { @@ -243,7 +245,7 @@ func (hg *Hypergraph) LookupAtom(a Atom) bool { } } -func (hg *Hypergraph) LookupAtomSet(atomSet map[[66]byte]Atom) bool { +func (hg *Hypergraph) LookupAtomSet(atomSet map[[64]byte]Atom) bool { for _, atom := range atomSet { if !hg.LookupAtom(atom) { return false diff --git a/node/hypergraph/application/hypergraph_convergence_test.go b/node/hypergraph/application/hypergraph_convergence_test.go index 1fe9f8a..81a7c55 100644 --- a/node/hypergraph/application/hypergraph_convergence_test.go +++ b/node/hypergraph/application/hypergraph_convergence_test.go @@ -22,9 +22,8 @@ func TestConvergence(t *testing.T) { vertices := make([]*application.Vertex, numOperations) for i := 0; i < numOperations; i++ { vertices[i] = &application.Vertex{ - AppAddress: [32]byte{byte(i % 256)}, - DataAddress: [32]byte{byte(i / 256)}, - SegmentOrder: uint16(i), + AppAddress: [32]byte{byte(i % 256)}, + DataAddress: [32]byte{byte(i / 256)}, } } @@ -33,7 +32,7 @@ func TestConvergence(t *testing.T) { hyperedges[i] = &application.Hyperedge{ AppAddress: [32]byte{byte(i % 256)}, DataAddress: [32]byte{byte(i / 256)}, - Extrinsics: make(map[[66]byte]application.Atom), + Extrinsics: make(map[[64]byte]application.Atom), } // Add some random vertices as extrinsics for j := 0; j < 3; j++ { diff --git a/node/hypergraph/application/hypergraph_test.go b/node/hypergraph/application/hypergraph_test.go index 84cb0a0..25f490d 100644 --- a/node/hypergraph/application/hypergraph_test.go +++ b/node/hypergraph/application/hypergraph_test.go @@ -12,8 +12,8 @@ func TestHypergraph(t *testing.T) { // Test vertex operations t.Run("Vertex Operations", func(t *testing.T) { - v1 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{1}, SegmentOrder: 1} - v2 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{2}, SegmentOrder: 1} + v1 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{1}} + v2 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{2}} // Add vertices err := hg.AddVertex(v1) @@ -48,15 +48,15 @@ func TestHypergraph(t *testing.T) { // Test hyperedge operations t.Run("Hyperedge Operations", func(t *testing.T) { - v3 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{1}, SegmentOrder: 1} - v4 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{2}, SegmentOrder: 1} + v3 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{1}} + v4 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{2}} hg.AddVertex(v3) hg.AddVertex(v4) h1 := &application.Hyperedge{ AppAddress: [32]byte{3}, DataAddress: [32]byte{1}, - Extrinsics: map[[66]byte]application.Atom{v3.GetID(): v3, v4.GetID(): v4}, + Extrinsics: map[[64]byte]application.Atom{v3.GetID(): v3, v4.GetID(): v4}, } // Add hyperedge @@ -82,15 +82,15 @@ func TestHypergraph(t *testing.T) { // Test "within" relationship t.Run("Within Relationship", func(t *testing.T) { - v5 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{1}, SegmentOrder: 1} - v6 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{2}, SegmentOrder: 1} + v5 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{1}} + v6 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{2}} hg.AddVertex(v5) hg.AddVertex(v6) h2 := &application.Hyperedge{ AppAddress: [32]byte{5}, DataAddress: [32]byte{1}, - Extrinsics: map[[66]byte]application.Atom{v5.GetID(): v5, v6.GetID(): v6}, + Extrinsics: map[[64]byte]application.Atom{v5.GetID(): v5, v6.GetID(): v6}, } hg.AddHyperedge(h2) @@ -101,7 +101,7 @@ func TestHypergraph(t *testing.T) { t.Error("v6 should be within h2") } - v7 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{3}, SegmentOrder: 1} + v7 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{3}} hg.AddVertex(v7) if hg.Within(v7, h2) { t.Error("v7 should not be within h2") @@ -110,20 +110,20 @@ func TestHypergraph(t *testing.T) { // Test nested hyperedges t.Run("Nested Hyperedges", func(t *testing.T) { - v8 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{1}, SegmentOrder: 1} - v9 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{2}, SegmentOrder: 1} + v8 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{1}} + v9 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{2}} hg.AddVertex(v8) hg.AddVertex(v9) h3 := &application.Hyperedge{ AppAddress: [32]byte{7}, DataAddress: [32]byte{1}, - Extrinsics: map[[66]byte]application.Atom{v8.GetID(): v8}, + Extrinsics: map[[64]byte]application.Atom{v8.GetID(): v8}, } h4 := &application.Hyperedge{ AppAddress: [32]byte{7}, DataAddress: [32]byte{2}, - Extrinsics: map[[66]byte]application.Atom{h3.GetID(): h3, v9.GetID(): v9}, + Extrinsics: map[[64]byte]application.Atom{h3.GetID(): h3, v9.GetID(): v9}, } hg.AddHyperedge(h3) hg.AddHyperedge(h4) @@ -138,11 +138,11 @@ func TestHypergraph(t *testing.T) { // Test error cases t.Run("Error Cases", func(t *testing.T) { - v10 := &application.Vertex{AppAddress: [32]byte{8}, DataAddress: [32]byte{1}, SegmentOrder: 1} + v10 := &application.Vertex{AppAddress: [32]byte{8}, DataAddress: [32]byte{1}} h5 := &application.Hyperedge{ AppAddress: [32]byte{8}, DataAddress: [32]byte{2}, - Extrinsics: map[[66]byte]application.Atom{v10.GetID(): v10}, + Extrinsics: map[[64]byte]application.Atom{v10.GetID(): v10}, } // Try to add hyperedge with non-existent vertex @@ -164,8 +164,8 @@ func TestHypergraph(t *testing.T) { // Test sharding t.Run("Sharding", func(t *testing.T) { - v11 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{1}, SegmentOrder: 1} - v12 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{2}, SegmentOrder: 1} + v11 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{1}} + v12 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{2}} hg.AddVertex(v11) hg.AddVertex(v12) diff --git a/node/test.sh b/node/test.sh index dc07696..9c88bbe 100755 --- a/node/test.sh +++ b/node/test.sh @@ -11,6 +11,6 @@ BINARIES_DIR="$ROOT_DIR/target/release" # Link the native VDF and execute tests pushd "$NODE_DIR" > /dev/null - CGO_LDFLAGS="-L$BINARIES_DIR -lvdf -lbls48581 -ldl" \ + CGO_LDFLAGS="-L$BINARIES_DIR -L/opt/homebrew/Cellar/mpfr/4.2.1/lib -I/opt/homebrew/Cellar/mpfr/4.2.1/include -L/opt/homebrew/Cellar/gmp/6.3.0/lib -I/opt/homebrew/Cellar/gmp/6.3.0/include -L/opt/homebrew/Cellar/flint/3.1.3-p1/lib -I/opt/homebrew/Cellar/flint/3.1.3-p1/include -lbls48581 -lstdc++ -lvdf -ldl -lm -lflint -lgmp -lmpfr" \ CGO_ENABLED=1 \ go test "$@" From 5ed5f4ca6ac2d3cc0c2a037f19c1b0cd784aff76 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Tue, 28 Jan 2025 08:35:48 -0600 Subject: [PATCH 02/13] resolve state tree issue, ensure message validation is consistent with mint --- node/consensus/data/consensus_frames.go | 1 + node/consensus/data/message_validators.go | 2 +- node/crypto/proof_tree.go | 82 ++++++++++--------- node/crypto/proof_tree_test.go | 71 +++++++++++++--- .../token/application/token_application.go | 2 +- .../token/token_execution_engine.go | 6 +- 6 files changed, 109 insertions(+), 55 deletions(-) diff --git a/node/consensus/data/consensus_frames.go b/node/consensus/data/consensus_frames.go index 873374c..f91fd98 100644 --- a/node/consensus/data/consensus_frames.go +++ b/node/consensus/data/consensus_frames.go @@ -74,6 +74,7 @@ func (e *DataClockConsensusEngine) syncWithMesh() error { func (e *DataClockConsensusEngine) prove( previousFrame *protobufs.ClockFrame, ) (*protobufs.ClockFrame, error) { + time.Sleep(40 * time.Second) if e.lastProven >= previousFrame.FrameNumber && e.lastProven != 0 { return previousFrame, nil } diff --git a/node/consensus/data/message_validators.go b/node/consensus/data/message_validators.go index f06596d..f2eef1c 100644 --- a/node/consensus/data/message_validators.go +++ b/node/consensus/data/message_validators.go @@ -117,7 +117,7 @@ func (e *DataClockConsensusEngine) validateTxMessage(peerID peer.ID, message *pb if err != nil { panic(err) } - if frameNumber+2 < head.FrameNumber { + if frameNumber+1 < head.FrameNumber { return p2p.ValidationResultIgnore } } diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index 73d50f4..61780d2 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -23,7 +23,7 @@ const ( ) type VectorCommitmentNode interface { - Commit() []byte + Commit(recalculate bool) []byte } type VectorCommitmentLeafNode struct { @@ -38,8 +38,8 @@ type VectorCommitmentBranchNode struct { Commitment []byte } -func (n *VectorCommitmentLeafNode) Commit() []byte { - if n.Commitment == nil { +func (n *VectorCommitmentLeafNode) Commit(recalculate bool) []byte { + if n.Commitment == nil || recalculate { h := sha512.New() h.Write([]byte{0}) h.Write(n.Key) @@ -49,12 +49,12 @@ func (n *VectorCommitmentLeafNode) Commit() []byte { return n.Commitment } -func (n *VectorCommitmentBranchNode) Commit() []byte { - if n.Commitment == nil { +func (n *VectorCommitmentBranchNode) Commit(recalculate bool) []byte { + if n.Commitment == nil || recalculate { data := []byte{} for _, child := range n.Children { if child != nil { - out := child.Commit() + out := child.Commit(recalculate) switch c := child.(type) { case *VectorCommitmentBranchNode: h := sha512.New() @@ -84,7 +84,7 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool { if n.Commitment == nil { for _, child := range n.Children { if child != nil { - out := child.Commit() + out := child.Commit(false) switch c := child.(type) { case *VectorCommitmentBranchNode: h := sha512.New() @@ -108,7 +108,7 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool { } else { child := n.Children[index] if child != nil { - out := child.Commit() + out := child.Commit(false) switch c := child.(type) { case *VectorCommitmentBranchNode: h := sha512.New() @@ -134,7 +134,7 @@ func (n *VectorCommitmentBranchNode) Prove(index int) []byte { data := []byte{} for _, child := range n.Children { if child != nil { - out := child.Commit() + out := child.Commit(false) switch c := child.(type) { case *VectorCommitmentBranchNode: h := sha512.New() @@ -416,13 +416,14 @@ func (t *VectorCommitmentTree) Delete(key []byte) error { return errors.New("empty key not allowed") } - var delete func(node VectorCommitmentNode, depth int) VectorCommitmentNode - delete = func(node VectorCommitmentNode, depth int) VectorCommitmentNode { + var remove func(node VectorCommitmentNode, depth int) VectorCommitmentNode + remove = func(node VectorCommitmentNode, depth int) VectorCommitmentNode { if node == nil { return nil } switch n := node.(type) { + case *VectorCommitmentLeafNode: if bytes.Equal(n.Key, key) { return nil @@ -430,63 +431,66 @@ func (t *VectorCommitmentTree) Delete(key []byte) error { return n case *VectorCommitmentBranchNode: - // Check prefix match for i, expectedNibble := range n.Prefix { currentNibble := getNextNibble(key, depth+i*BranchBits) if currentNibble != expectedNibble { - return n // Key doesn't match prefix, nothing to delete + return n } } - // Delete at final position after prefix finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) - n.Children[finalNibble] = delete(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) + n.Children[finalNibble] = + remove(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) + n.Commitment = nil - // Count remaining children childCount := 0 var lastChild VectorCommitmentNode - var lastIndex int + var lastChildIndex int for i, child := range n.Children { if child != nil { childCount++ lastChild = child - lastIndex = i + lastChildIndex = i } } - if childCount == 0 { + switch childCount { + case 0: return nil - } else if childCount == 1 { - // If the only child is a leaf, keep structure if its path matches - if leaf, ok := lastChild.(*VectorCommitmentLeafNode); ok { - if lastIndex == getLastNibble(leaf.Key, len(n.Prefix)) { - return n - } - return leaf - } - // If it's a branch, merge the prefixes - if branch, ok := lastChild.(*VectorCommitmentBranchNode); ok { - branch.Prefix = append(n.Prefix, branch.Prefix...) - return branch - } - } - return n - } + case 1: + if childBranch, ok := lastChild.(*VectorCommitmentBranchNode); ok { + // Merge: + // n.Prefix + [lastChildIndex] + childBranch.Prefix + mergedPrefix := make([]int, 0, len(n.Prefix)+1+len(childBranch.Prefix)) + mergedPrefix = append(mergedPrefix, n.Prefix...) + mergedPrefix = append(mergedPrefix, lastChildIndex) + mergedPrefix = append(mergedPrefix, childBranch.Prefix...) - return nil + childBranch.Prefix = mergedPrefix + childBranch.Commitment = nil + return childBranch + } + + return lastChild + default: + return n + } + default: + return node + } } - t.Root = delete(t.Root, 0) + t.Root = remove(t.Root, 0) return nil } // Commit returns the root of the tree -func (t *VectorCommitmentTree) Commit() []byte { +func (t *VectorCommitmentTree) Commit(recalculate bool) []byte { if t.Root == nil { return make([]byte, 64) } - return t.Root.Commit() + return t.Root.Commit(recalculate) } func debugNode(node VectorCommitmentNode, depth int, prefix string) { diff --git a/node/crypto/proof_tree_test.go b/node/crypto/proof_tree_test.go index c7f186b..f855398 100644 --- a/node/crypto/proof_tree_test.go +++ b/node/crypto/proof_tree_test.go @@ -36,7 +36,7 @@ func BenchmarkVectorCommitmentTreeCommit(b *testing.B) { if err != nil { b.Errorf("Failed to insert item %d: %v", i, err) } - tree.Commit() + tree.Commit(false) } } @@ -221,7 +221,7 @@ func TestVectorCommitmentTrees(t *testing.T) { // Root should change after insert tree.Insert([]byte("key1"), []byte("value1")) - firstRoot := tree.Root.Commit() + firstRoot := tree.Root.Commit(false) if bytes.Equal(firstRoot, bytes.Repeat([]byte{0x00}, 64)) { t.Error("Root hash should change after insert") @@ -229,7 +229,7 @@ func TestVectorCommitmentTrees(t *testing.T) { // Root should change after update tree.Insert([]byte("key1"), []byte("value2")) - secondRoot := tree.Root.Commit() + secondRoot := tree.Root.Commit(false) if bytes.Equal(secondRoot, firstRoot) { t.Error("Root hash should change after update") @@ -248,14 +248,27 @@ func TestVectorCommitmentTrees(t *testing.T) { addresses := [][]byte{} - for i := 0; i < 1000; i++ { + for i := 0; i < 10000; i++ { d := make([]byte, 32) rand.Read(d) addresses = append(addresses, d) } - // Insert 1000 items - for i := 0; i < 1000; i++ { + kept := [][]byte{} + for i := 0; i < 5000; i++ { + kept = append(kept, addresses[i]) + } + + newAdditions := [][]byte{} + for i := 0; i < 5000; i++ { + d := make([]byte, 32) + rand.Read(d) + newAdditions = append(newAdditions, d) + kept = append(kept, d) + } + + // Insert 10000 items + for i := 0; i < 10000; i++ { key := addresses[i] value := addresses[i] err := tree.Insert(key, value) @@ -264,8 +277,8 @@ func TestVectorCommitmentTrees(t *testing.T) { } } - // Insert 1000 items in reverse - for i := 999; i >= 0; i-- { + // Insert 10000 items in reverse + for i := 9999; i >= 0; i-- { key := addresses[i] value := addresses[i] err := cmptree.Insert(key, value) @@ -275,7 +288,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Verify all items - for i := 0; i < 1000; i++ { + for i := 0; i < 10000; i++ { key := addresses[i] expected := addresses[i] value, err := tree.Get(key) @@ -294,8 +307,44 @@ func TestVectorCommitmentTrees(t *testing.T) { } } - tcommit := tree.Root.Commit() - cmptcommit := cmptree.Root.Commit() + // delete keys + for i := 5000; i < 10000; i++ { + key := addresses[i] + fmt.Printf("delete %x\n", key) + tree.Delete(key) + } + + // add new + for i := 0; i < 5000; i++ { + tree.Insert(newAdditions[i], newAdditions[i]) + } + + cmptree = &VectorCommitmentTree{} + + for i := 0; i < 10000; i++ { + cmptree.Insert(kept[i], kept[i]) + } + // Verify all items + for i := 0; i < 10000; i++ { + key := kept[i] + expected := kept[i] + value, err := tree.Get(key) + if err != nil { + t.Errorf("Failed to get item %d: %v", i, err) + } + cmpvalue, err := cmptree.Get(key) + if err != nil { + t.Errorf("Failed to get item %d: %v", i, err) + } + if !bytes.Equal(value, expected) { + t.Errorf("Item %d: expected %x, got %x", i, string(expected), string(value)) + } + if !bytes.Equal(expected, cmpvalue) { + t.Errorf("Item %d: expected %x, got %x", i, string(value), string(cmpvalue)) + } + } + tcommit := tree.Root.Commit(false) + cmptcommit := cmptree.Root.Commit(false) if !bytes.Equal(tcommit, cmptcommit) { t.Errorf("tree mismatch, %x, %x", tcommit, cmptcommit) diff --git a/node/execution/intrinsics/token/application/token_application.go b/node/execution/intrinsics/token/application/token_application.go index d6e4c54..8424152 100644 --- a/node/execution/intrinsics/token/application/token_application.go +++ b/node/execution/intrinsics/token/application/token_application.go @@ -209,7 +209,7 @@ func (a *TokenApplication) ApplyTransitions( continue } else if len(t.Mint.Proofs) >= 3 && currentFrameNumber > PROOF_FRAME_CUTOFF { frameNumber := binary.BigEndian.Uint64(t.Mint.Proofs[2]) - if frameNumber < currentFrameNumber-2 { + if frameNumber < currentFrameNumber-1 { fails[i] = transition continue } diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index f0f35d9..2bafaed 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -348,7 +348,7 @@ func NewTokenExecutionEngine( e.provingKeyAddress = provingKeyAddress frame, _, err := e.clockStore.GetLatestDataClockFrame(e.intrinsicFilter) - if err != nil || frame.FrameNumber < 184479 { + if err != nil || frame.FrameNumber < 186405 { e.rebuildStateTree() } else { e.stateTree, err = e.clockStore.GetDataStateTree(e.intrinsicFilter) @@ -457,7 +457,7 @@ func (e *TokenExecutionEngine) rebuildStateTree() { e.logger.Info("committing state tree") - root := e.stateTree.Commit() + root := e.stateTree.Commit(true) e.logger.Info( "committed state tree", @@ -1049,7 +1049,7 @@ func (e *TokenExecutionEngine) ProcessFrame( e.logger.Info("committing state tree") - root := stateTree.Commit() + root := stateTree.Commit(false) e.logger.Info( "commited state tree", From b5fd0775bfb88192c633cc575588df6c3e6a0dda Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 30 Jan 2025 00:24:45 -0600 Subject: [PATCH 03/13] prepare new genesis for next stage of tests --- node/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/config/config.go b/node/config/config.go index 18da56e..30639a6 100644 --- a/node/config/config.go +++ b/node/config/config.go @@ -165,7 +165,7 @@ var unlock *SignedGenesisUnlock func DownloadAndVerifyGenesis(network uint) (*SignedGenesisUnlock, error) { if network != 0 { unlock = &SignedGenesisUnlock{ - GenesisSeedHex: "726573697374206d7563682c206f626579206c6974746c657c00000000000000000000000C", + GenesisSeedHex: "726573697374206d7563682c206f626579206c6974746c657c00000000000000000000000D", Beacon: []byte{ 0x58, 0xef, 0xd9, 0x7e, 0xdd, 0x0e, 0xb6, 0x2f, 0x51, 0xc7, 0x5d, 0x00, 0x29, 0x12, 0x45, 0x49, From 772edd31be5042d83354a561c15b7bf997476d90 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 30 Jan 2025 02:20:01 -0600 Subject: [PATCH 04/13] handle forked genesis --- node/consensus/data/consensus_frames.go | 19 +++++++++++++++++++ .../token/application/token_application.go | 3 +++ 2 files changed, 22 insertions(+) diff --git a/node/consensus/data/consensus_frames.go b/node/consensus/data/consensus_frames.go index f91fd98..5a1d666 100644 --- a/node/consensus/data/consensus_frames.go +++ b/node/consensus/data/consensus_frames.go @@ -411,22 +411,41 @@ func (e *DataClockConsensusEngine) syncWithPeer( zap.Uint64("frame_number", response.ClockFrame.FrameNumber), zap.Duration("frame_age", frametime.Since(response.ClockFrame)), ) + if !e.IsInProverTrie( response.ClockFrame.GetPublicKeySignatureEd448().PublicKey.KeyValue, ) { cooperative = false } + if err := e.frameProver.VerifyDataClockFrame( response.ClockFrame, ); err != nil { return latest, doneChs, errors.Wrap(err, "sync") } + + // Useful for testnet, immediately handles equivocation from multiple + // genesis events: + if response.ClockFrame.FrameNumber == 1 { + genesis, _, _ := e.clockStore.GetDataClockFrame(e.filter, 0, true) + selector, _ := genesis.GetSelector() + if !bytes.Equal( + response.ClockFrame.ParentSelector, + selector.FillBytes(make([]byte, 32)), + ) { + cooperative = false + return latest, doneChs, errors.Wrap(errors.New("invalid frame"), "sync") + } + } + doneCh, err := e.dataTimeReel.Insert(e.ctx, response.ClockFrame) if err != nil { return latest, doneChs, errors.Wrap(err, "sync") } + doneChs = append(doneChs, doneCh) latest = response.ClockFrame + if latest.FrameNumber >= maxFrame { return latest, doneChs, nil } diff --git a/node/execution/intrinsics/token/application/token_application.go b/node/execution/intrinsics/token/application/token_application.go index 8424152..bb50ef3 100644 --- a/node/execution/intrinsics/token/application/token_application.go +++ b/node/execution/intrinsics/token/application/token_application.go @@ -414,6 +414,9 @@ func (a *TokenApplication) ApplyTransitions( wg.Wait() for i, transition := range set { + if transition == nil { + continue + } if fails[i] != nil { continue } From 7f8137df6789bc82486b6a947dcf5249626fe6f0 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 30 Jan 2025 03:20:52 -0600 Subject: [PATCH 05/13] reset cutoffs --- node/consensus/data/main_data_loop.go | 4 ++++ .../intrinsics/token/application/token_handle_mint.go | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/node/consensus/data/main_data_loop.go b/node/consensus/data/main_data_loop.go index 15f8158..7f1b342 100644 --- a/node/consensus/data/main_data_loop.go +++ b/node/consensus/data/main_data_loop.go @@ -120,6 +120,10 @@ outer: continue } + if head.FrameNumber <= maxFrames { + continue + } + to := head.FrameNumber - maxFrames for i := from; i < to; i += batchSize { start, stop := i, min(i+batchSize, to) diff --git a/node/execution/intrinsics/token/application/token_handle_mint.go b/node/execution/intrinsics/token/application/token_handle_mint.go index d97eb1e..9936ed5 100644 --- a/node/execution/intrinsics/token/application/token_handle_mint.go +++ b/node/execution/intrinsics/token/application/token_handle_mint.go @@ -19,12 +19,12 @@ import ( ) // for tests, these need to be var -var PROOF_FRAME_CUTOFF = uint64(46500) -var PROOF_FRAME_RING_RESET = uint64(52000) -var PROOF_FRAME_RING_RESET_2 = uint64(53028) -var PROOF_FRAME_COMBINE_CUTOFF = uint64(162000) +var PROOF_FRAME_CUTOFF = uint64(0) +var PROOF_FRAME_RING_RESET = uint64(0) +var PROOF_FRAME_RING_RESET_2 = uint64(0) +var PROOF_FRAME_COMBINE_CUTOFF = uint64(0) -const PROOF_FRAME_SENIORITY_REPAIR = 59029 +const PROOF_FRAME_SENIORITY_REPAIR = 0 type processedMint struct { isPre2 bool From 6df30fb455c1f0f0e1352daa8c6ce64fa7678089 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Fri, 31 Jan 2025 01:53:32 -0600 Subject: [PATCH 06/13] resolve tree reinit bug --- node/crypto/proof_tree.go | 1 + node/crypto/tree_compare.go | 258 ++++++++++++++++++ .../application/token_handle_prover_join.go | 3 + .../token/token_execution_engine.go | 12 +- 4 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 node/crypto/tree_compare.go diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index 61780d2..a712e12 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -262,6 +262,7 @@ func (t *VectorCommitmentTree) Insert(key, value []byte) error { return newBranch } } + // Key matches prefix, continue with final nibble finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) n.Children[finalNibble] = insert(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) diff --git a/node/crypto/tree_compare.go b/node/crypto/tree_compare.go new file mode 100644 index 0000000..4893647 --- /dev/null +++ b/node/crypto/tree_compare.go @@ -0,0 +1,258 @@ +package crypto + +import ( + "bytes" + "fmt" +) + +// CompareTreesAtHeight compares two vector commitment trees at each level +func CompareTreesAtHeight(tree1, tree2 *VectorCommitmentTree) [][]ComparisonResult { + if tree1 == nil || tree2 == nil { + return nil + } + + var results [][]ComparisonResult + maxHeight := getMaxHeight(tree1.Root, tree2.Root) + + // Compare level by level + for height := 0; height <= maxHeight; height++ { + levelResults := compareLevelCommits(tree1.Root, tree2.Root, height, 0) + results = append(results, levelResults) + } + + return results +} + +type ComparisonResult struct { + Path []int // Path taken to reach this node (nibble values) + Height int // Current height in the tree + Commit1 []byte // Commitment from first tree + Commit2 []byte // Commitment from second tree + Matches bool // Whether the commitments match +} + +func getMaxHeight(node1, node2 VectorCommitmentNode) int { + height1 := getHeight(node1) + height2 := getHeight(node2) + if height1 > height2 { + return height1 + } + return height2 +} + +func getHeight(node VectorCommitmentNode) int { + if node == nil { + return 0 + } + + switch n := node.(type) { + case *VectorCommitmentLeafNode: + return 0 + case *VectorCommitmentBranchNode: + maxChildHeight := 0 + for _, child := range n.Children { + childHeight := getHeight(child) + if childHeight > maxChildHeight { + maxChildHeight = childHeight + } + } + return maxChildHeight + 1 + len(n.Prefix) + } + return 0 +} + +func compareLevelCommits(node1, node2 VectorCommitmentNode, targetHeight, currentHeight int) []ComparisonResult { + if node1 == nil && node2 == nil { + return nil + } + + // If we've reached the target height, compare the commits + if currentHeight == targetHeight { + var commit1, commit2 []byte + if node1 != nil { + commit1 = node1.Commit(false) + } + if node2 != nil { + commit2 = node2.Commit(false) + } + + return []ComparisonResult{{ + Height: targetHeight, + Commit1: commit1, + Commit2: commit2, + Matches: bytes.Equal(commit1, commit2), + }} + } + + // If we haven't reached the target height, traverse deeper + var results []ComparisonResult + + // Handle branch nodes + switch n1 := node1.(type) { + case *VectorCommitmentBranchNode: + n2, ok := node2.(*VectorCommitmentBranchNode) + if !ok { + // Trees have different structure at this point + return results + } + + // Account for prefix lengths + nextHeight := currentHeight + if len(n1.Prefix) > 0 { + nextHeight += len(n1.Prefix) + } + + // If we're still below target height after prefix, traverse children + if nextHeight < targetHeight { + for i := 0; i < BranchNodes; i++ { + childResults := compareLevelCommits(n1.Children[i], n2.Children[i], targetHeight, nextHeight+1) + results = append(results, childResults...) + } + } + } + + return results +} + +// TraverseAndCompare provides a channel-based iterator for comparing trees +func TraverseAndCompare(tree1, tree2 *VectorCommitmentTree) chan ComparisonResult { + resultChan := make(chan ComparisonResult) + + go func() { + defer close(resultChan) + + if tree1 == nil || tree2 == nil { + return + } + + maxHeight := getMaxHeight(tree1.Root, tree2.Root) + + // Traverse each height + for height := 0; height <= maxHeight; height++ { + results := compareLevelCommits(tree1.Root, tree2.Root, height, 0) + for _, result := range results { + resultChan <- result + } + } + }() + + return resultChan +} + +// Example usage: +// LeafDifference contains information about leaves that differ between trees +type LeafDifference struct { + Key []byte // The key of the leaf + OnlyInTree1 bool // True if the leaf only exists in tree1 + OnlyInTree2 bool // True if the leaf only exists in tree2 + Value1 []byte // Value from tree1 (if present) + Value2 []byte // Value from tree2 (if present) +} + +// CompareLeaves returns all leaves that differ between the two trees +func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { + // Get all leaves from both trees + leaves1 := getAllLeaves(tree1.Root) + leaves2 := getAllLeaves(tree2.Root) + + differences := make([]LeafDifference, 0) + + // Use maps for efficient lookup + leafMap1 := make(map[string]*VectorCommitmentLeafNode) + leafMap2 := make(map[string]*VectorCommitmentLeafNode) + + // Build maps + for _, leaf := range leaves1 { + leafMap1[string(leaf.Key)] = leaf + } + for _, leaf := range leaves2 { + leafMap2[string(leaf.Key)] = leaf + } + + // Find leaves only in tree1 or with different values + for _, leaf1 := range leaves1 { + key := string(leaf1.Key) + if leaf2, exists := leafMap2[key]; exists { + // Leaf exists in both trees, check if values match + if !bytes.Equal(leaf1.Value, leaf2.Value) { + differences = append(differences, LeafDifference{ + Key: leaf1.Key, + Value1: leaf1.Value, + Value2: leaf2.Value, + }) + } + } else { + // Leaf only exists in tree1 + differences = append(differences, LeafDifference{ + Key: leaf1.Key, + OnlyInTree1: true, + Value1: leaf1.Value, + }) + } + } + + // Find leaves only in tree2 + for _, leaf2 := range leaves2 { + key := string(leaf2.Key) + if _, exists := leafMap1[key]; !exists { + differences = append(differences, LeafDifference{ + Key: leaf2.Key, + OnlyInTree2: true, + Value2: leaf2.Value, + }) + } + } + + return differences +} + +// getAllLeaves returns all leaf nodes in the tree +func getAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { + if node == nil { + return nil + } + + var leaves []*VectorCommitmentLeafNode + + switch n := node.(type) { + case *VectorCommitmentLeafNode: + leaves = append(leaves, n) + case *VectorCommitmentBranchNode: + for _, child := range n.Children { + if child != nil { + childLeaves := getAllLeaves(child) + leaves = append(leaves, childLeaves...) + } + } + } + + return leaves +} + +func ExampleComparison() { + // Create and populate two trees + tree1 := &VectorCommitmentTree{} + tree2 := &VectorCommitmentTree{} + + // Compare trees using channel-based iterator + for result := range TraverseAndCompare(tree1, tree2) { + if !result.Matches { + fmt.Printf("Mismatch at height %d\n", result.Height) + fmt.Printf("Tree1 commit: %x\n", result.Commit1) + fmt.Printf("Tree2 commit: %x\n", result.Commit2) + } + } + + // Compare leaves between trees + differences := CompareLeaves(tree1, tree2) + for _, diff := range differences { + if diff.OnlyInTree1 { + fmt.Printf("Key %x only exists in tree1 with value %x\n", diff.Key, diff.Value1) + } else if diff.OnlyInTree2 { + fmt.Printf("Key %x only exists in tree2 with value %x\n", diff.Key, diff.Value2) + } else { + fmt.Printf("Key %x has different values: tree1=%x, tree2=%x\n", + diff.Key, diff.Value1, diff.Value2) + } + } +} diff --git a/node/execution/intrinsics/token/application/token_handle_prover_join.go b/node/execution/intrinsics/token/application/token_handle_prover_join.go index c3ee972..1e03b7e 100644 --- a/node/execution/intrinsics/token/application/token_handle_prover_join.go +++ b/node/execution/intrinsics/token/application/token_handle_prover_join.go @@ -45,10 +45,12 @@ func (a *TokenApplication) handleDataAnnounceProverJoin( error, ) { if currentFrameNumber < PROOF_FRAME_CUTOFF { + a.Logger.Debug("join earlier than cutoff", zap.Uint64("current_frame", currentFrameNumber), zap.Uint64("cutoff", PROOF_FRAME_CUTOFF)) return nil, errors.Wrap(ErrInvalidStateTransition, "handle join") } if err := t.Validate(); err != nil { + a.Logger.Debug("invalid join", zap.Error(err)) return nil, errors.Wrap(ErrInvalidStateTransition, "handle join") } @@ -94,6 +96,7 @@ func (a *TokenApplication) handleDataAnnounceProverJoin( if t.Announce != nil { outputs, err = a.handleAnnounce(currentFrameNumber, lockMap, t.Announce) if err != nil { + a.Logger.Debug("bad announce", zap.Error(err)) return nil, errors.Wrap(ErrInvalidStateTransition, "handle join") } } diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 2bafaed..289af7e 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -436,7 +436,11 @@ func (e *TokenExecutionEngine) rebuildStateTree() { panic(err) } for iter.First(); iter.Valid(); iter.Next() { - e.stateTree.Insert(iter.Key()[2:], iter.Value()) + key := make([]byte, len(iter.Key()[2:])) + value := make([]byte, len(iter.Value())) + copy(key, iter.Key()[2:]) + copy(value, iter.Value()) + e.stateTree.Insert(key, value) } iter.Close() @@ -445,7 +449,11 @@ func (e *TokenExecutionEngine) rebuildStateTree() { panic(err) } for iter.First(); iter.Valid(); iter.Next() { - e.stateTree.Insert(iter.Key()[2:], iter.Value()) + key := make([]byte, len(iter.Key()[2:])) + value := make([]byte, len(iter.Value())) + copy(key, iter.Key()[2:]) + copy(value, iter.Value()) + e.stateTree.Insert(key, value) } iter.Close() e.logger.Info("saving rebuilt state tree") From 4696d5292cd77d6d09f1bfdd0f208195ec39a862 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 6 Feb 2025 06:34:22 -0600 Subject: [PATCH 07/13] restore verenc into hypergraph loop, thread hypergraph through token application for backcompat --- Cargo.lock | 25 + Cargo.toml | 1 + Dockerfile.source | 4 + crates/verenc/.gitignore | 10 + crates/verenc/Cargo.toml | 22 + crates/verenc/README.md | 28 + crates/verenc/build.rs | 5 + crates/verenc/run_size_benchmarks.sh | 7 + crates/verenc/src/lib.rs | 526 +++++++++++++++ crates/verenc/src/lib.udl | 55 ++ crates/verenc/src/pke.rs | 116 ++++ crates/verenc/src/rdkgith.rs | 531 +++++++++++++++ crates/verenc/src/seed_tree.rs | 199 ++++++ crates/verenc/src/utils.rs | 189 ++++++ crates/verenc/src/ve.rs | 30 + node/app/wire.go | 2 + node/app/wire_gen.go | 8 +- node/build.sh | 4 +- node/consensus/data/peer_messaging.go | 6 - node/consensus/data/token_handle_mint_test.go | 9 +- node/crypto/proof_tree.go | 134 +++- node/crypto/proof_tree_test.go | 58 +- node/crypto/tree_compare.go | 10 +- node/crypto/verifiable_encryption.go | 231 +++++++ .../token_handle_prover_join_test.go | 33 +- .../token/token_execution_engine.go | 193 +++++- .../intrinsics/token/token_genesis.go | 57 +- node/go.mod | 4 + node/go.sum | 1 + node/hypergraph/application/hypergraph.go | 629 ++++++++++++++---- .../hypergraph_convergence_test.go | 58 +- .../hypergraph/application/hypergraph_test.go | 71 +- node/main.go | 554 --------------- node/store/coin.go | 25 - node/store/hypergraph.go | 238 +++++++ node/test.sh | 2 +- verenc/.gitignore | 1 + verenc/README.md | 9 + verenc/generate.sh | 14 + verenc/go.mod | 8 + verenc/go.sum | 4 + verenc/test.sh | 16 + verenc/verenc.go | 35 + verenc/verenc_test.go | 102 +++ 44 files changed, 3385 insertions(+), 879 deletions(-) create mode 100644 crates/verenc/.gitignore create mode 100644 crates/verenc/Cargo.toml create mode 100644 crates/verenc/README.md create mode 100644 crates/verenc/build.rs create mode 100755 crates/verenc/run_size_benchmarks.sh create mode 100644 crates/verenc/src/lib.rs create mode 100644 crates/verenc/src/lib.udl create mode 100644 crates/verenc/src/pke.rs create mode 100644 crates/verenc/src/rdkgith.rs create mode 100644 crates/verenc/src/seed_tree.rs create mode 100644 crates/verenc/src/utils.rs create mode 100644 crates/verenc/src/ve.rs create mode 100644 node/crypto/verifiable_encryption.go create mode 100644 node/store/hypergraph.go create mode 100644 verenc/.gitignore create mode 100644 verenc/README.md create mode 100755 verenc/generate.sh create mode 100644 verenc/go.mod create mode 100644 verenc/go.sum create mode 100755 verenc/test.sh create mode 100644 verenc/verenc.go create mode 100644 verenc/verenc_test.go diff --git a/Cargo.lock b/Cargo.lock index 432e783..01be94c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1517,6 +1517,19 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.1", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1834,6 +1847,18 @@ dependencies = [ "uniffi", ] +[[package]] +name = "verenc" +version = "0.1.0" +dependencies = [ + "ed448-goldilocks-plus", + "hex 0.4.3", + "rand", + "serde", + "sha2 0.9.9", + "uniffi", +] + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 13ef076..5d6d993 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "crates/bls48581", "crates/ed448-rust", "crates/rpm", + "crates/verenc", ] [profile.release] diff --git a/Dockerfile.source b/Dockerfile.source index 1a34735..f5e08b1 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -76,6 +76,10 @@ RUN ./generate.sh WORKDIR /opt/ceremonyclient/bls48581 RUN ./generate.sh +## Generate Rust bindings for VerEnc +WORKDIR /opt/ceremonyclient/verenc +RUN ./generate.sh + # Build and install the node WORKDIR /opt/ceremonyclient/node diff --git a/crates/verenc/.gitignore b/crates/verenc/.gitignore new file mode 100644 index 0000000..088ba6b --- /dev/null +++ b/crates/verenc/.gitignore @@ -0,0 +1,10 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk diff --git a/crates/verenc/Cargo.toml b/crates/verenc/Cargo.toml new file mode 100644 index 0000000..7c38269 --- /dev/null +++ b/crates/verenc/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "verenc" +version = "0.1.0" +authors = ["Anonymous conference submission", "Cassandra Heart"] +edition = "2018" +description = "Implementation of verifiable encryption of discrete logarithms with the DKG-in-the-head approach, adapted to Curve448" +license = "MIT" + +[lib] +crate-type = ["lib", "staticlib"] +name = "verenc" + +[dependencies] +sha2 = "0.9.0" +hex = "0.4.0" +rand = "0.8" +ed448-goldilocks-plus = "0.11.2" +uniffi = { version= "0.25", features = ["cli"]} +serde = "1.0.208" + +[build-dependencies] +uniffi = { version = "0.25", features = [ "build" ] } \ No newline at end of file diff --git a/crates/verenc/README.md b/crates/verenc/README.md new file mode 100644 index 0000000..f763493 --- /dev/null +++ b/crates/verenc/README.md @@ -0,0 +1,28 @@ + +## Introduction +Implementation of the DKG-in-the-head (DKGitH) and Robust DKG-in-the-head (RDKGitH) verifiable encryption schemes. + +## Description +These verifiable encryption (VE) schemes allow one to encrypt a discrete logarithm instance under an Elgamal public key and prove to anyone that the correct value is encrypted. + +We use the elliptic curve implementation of [`arkworks`](https://github.com/arkworks-rs), and our implementation defaults to using the `secp256r1` curve, but is generic over the choice of curve and can easily be modified used to other curves implemented in `arkworks`. + +Hashing is done with SHA512, using the Rust [`sha2`](https://docs.rs/sha2/latest/sha2/) crate. + +Our seed tree implementation is inspired by the one in the C implementation of the [LegRoast](https://github.com/WardBeullens/LegRoast) signature scheme. + +## Running Tests and Benchmarks +To run unit tests type `cargo test --release`. + +Sizes of the proofs and ciphertexts for the two schemes are computed in unit tests, use the script `run_size_benchmarks.sh` to run the tests and display the output. + +Benchmarks of the time required to run the main VE operations `Prove()`, `Verify()`, `Compress()` and `Recover()` +are also provided, and can be run with `cargo bench`. To run only the DKGitH benchmarks, use +``` +cargo bench -- "^DKGitH" +``` +and to run only the RDKGitH benchmarks use +``` +cargo bench -- "^RDKGitH" +``` + diff --git a/crates/verenc/build.rs b/crates/verenc/build.rs new file mode 100644 index 0000000..f827f5c --- /dev/null +++ b/crates/verenc/build.rs @@ -0,0 +1,5 @@ +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + uniffi::generate_scaffolding("src/lib.udl").expect("uniffi generation failed"); +} diff --git a/crates/verenc/run_size_benchmarks.sh b/crates/verenc/run_size_benchmarks.sh new file mode 100755 index 0000000..a275273 --- /dev/null +++ b/crates/verenc/run_size_benchmarks.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "------------------------------------"; +echo "Printing sizes for RDKGitH instances:"; +echo "------------------------------------"; +cargo test --release -- --exact --nocapture rdkgith::tests::test_ve_print_sizes; + diff --git a/crates/verenc/src/lib.rs b/crates/verenc/src/lib.rs new file mode 100644 index 0000000..96159bf --- /dev/null +++ b/crates/verenc/src/lib.rs @@ -0,0 +1,526 @@ +pub mod rdkgith; +pub mod utils; +pub mod pke; +pub mod ve; +pub mod seed_tree; + +use std::convert::TryFrom; +use std::convert::TryInto; + +use ed448_goldilocks_plus::CompressedEdwardsY; +use ed448_goldilocks_plus::EdwardsPoint; +use ed448_goldilocks_plus::Scalar; +use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; + +pub use crate::rdkgith::*; +pub use crate::utils::*; +pub use crate::pke::*; +pub use crate::ve::*; +pub use crate::seed_tree::*; + +uniffi::include_scaffolding!("lib"); + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct VerencCiphertext { + pub c1: Vec, + pub c2: Vec, + pub i: u64, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct VerencShare { + pub s1: Vec, + pub s2: Vec, + pub i: u64, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct VerencProofAndBlindingKey { + pub blinding_key: Vec, + pub blinding_pubkey: Vec, + pub decryption_key: Vec, + pub encryption_key: Vec, + pub statement: Vec, + pub challenge: Vec, + pub polycom: Vec>, + pub ctexts: Vec, + pub shares_rands: Vec, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct VerencDecrypt { + pub blinding_pubkey: Vec, + pub decryption_key: Vec, + pub statement: Vec, + pub ciphertexts: CompressedCiphertext, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct VerencProof { + pub blinding_pubkey: Vec, + pub encryption_key: Vec, + pub statement: Vec, + pub challenge: Vec, + pub polycom: Vec>, + pub ctexts: Vec, + pub shares_rands: Vec, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct CompressedCiphertext { + pub ctexts: Vec, + pub aux: Vec>, +} + +pub fn new_verenc_proof(data: Vec) -> VerencProofAndBlindingKey { + if data.len() != 56 { + return VerencProofAndBlindingKey{ + blinding_key: vec![], + blinding_pubkey: vec![], + decryption_key: vec![], + encryption_key: vec![], + statement: vec![], + challenge: vec![], + polycom: vec![], + ctexts: vec![], + shares_rands: vec![], + }; + } + + let blind = Scalar::random(&mut OsRng); + let params = CurveParams::init(EdwardsPoint::GENERATOR * blind); + let pke = Elgamal::setup(¶ms); + let (N, t, n) = RVE_PARAMS[0]; + let vparams = RDkgithParams{ N, t, n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + + let dk = ve.kgen(); + let (stm, wit) = ve.igen(&data.try_into().unwrap()); + let pi = ve.prove(&stm, &wit); + + return VerencProofAndBlindingKey { + blinding_key: blind.to_bytes().to_vec(), + blinding_pubkey: (EdwardsPoint::GENERATOR * blind).compress().to_bytes().to_vec(), + decryption_key: dk.to_bytes().to_vec(), + encryption_key: (EdwardsPoint::GENERATOR * dk).compress().to_bytes().to_vec(), + statement: stm.compress().to_bytes().to_vec(), + challenge: pi.challenge, + polycom: pi.polycom.iter().map(|p| p.compress().to_bytes().to_vec()).collect(), + ctexts: pi.ctexts.iter().map(|c| VerencCiphertext { + c1: c.0.c1.compress().to_bytes().to_vec(), + c2: c.0.c2.to_bytes().to_vec(), + i: c.1 as u64, + }).collect(), + shares_rands: pi.shares_rands.iter().map(|s| VerencShare { + s1: s.0.to_bytes().to_vec(), + s2: s.1.to_bytes().to_vec(), + i: s.2 as u64, + }).collect(), + }; +} + +pub fn new_verenc_proof_encrypt_only(data: Vec, encryption_key_bytes: Vec) -> VerencProofAndBlindingKey { + if data.len() != 56 { + return VerencProofAndBlindingKey{ + blinding_key: vec![], + blinding_pubkey: vec![], + decryption_key: vec![], + encryption_key: vec![], + statement: vec![], + challenge: vec![], + polycom: vec![], + ctexts: vec![], + shares_rands: vec![], + }; + } + + let encryption_key = point_from_bytes(encryption_key_bytes.clone()); + if encryption_key.is_none() { + return VerencProofAndBlindingKey{ + blinding_key: vec![], + blinding_pubkey: vec![], + decryption_key: vec![], + encryption_key: vec![], + statement: vec![], + challenge: vec![], + polycom: vec![], + ctexts: vec![], + shares_rands: vec![], + }; + } + + let blind = Scalar::random(&mut OsRng); + let params = CurveParams::init(EdwardsPoint::GENERATOR * blind); + let pke = Elgamal::setup(¶ms); + let (N, t, n) = RVE_PARAMS[0]; + let vparams = RDkgithParams{ N, t, n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + + ve.set_ek(PKEPublicKey{ + ek: encryption_key.unwrap(), + }); + let (stm, wit) = ve.igen(&data.try_into().unwrap()); + let pi = ve.prove(&stm, &wit); + + return VerencProofAndBlindingKey { + blinding_key: blind.to_bytes().to_vec(), + blinding_pubkey: (EdwardsPoint::GENERATOR * blind).compress().to_bytes().to_vec(), + decryption_key: vec![], + encryption_key: encryption_key_bytes.clone(), + statement: stm.compress().to_bytes().to_vec(), + challenge: pi.challenge, + polycom: pi.polycom.iter().map(|p| p.compress().to_bytes().to_vec()).collect(), + ctexts: pi.ctexts.iter().map(|c| VerencCiphertext { + c1: c.0.c1.compress().to_bytes().to_vec(), + c2: c.0.c2.to_bytes().to_vec(), + i: c.1 as u64, + }).collect(), + shares_rands: pi.shares_rands.iter().map(|s| VerencShare { + s1: s.0.to_bytes().to_vec(), + s2: s.1.to_bytes().to_vec(), + i: s.2 as u64, + }).collect(), + }; +} + +fn point_from_bytes(bytes: Vec) -> Option { + if bytes.len() != 57 { + return None; + } + + let key_bytes: Result<[u8; 57], _> = bytes.try_into(); + if key_bytes.is_err() { + return None; + } + + let compressed_key = CompressedEdwardsY::try_from(key_bytes.unwrap()); + if compressed_key.is_err() { + return None; + } + + let key = compressed_key.unwrap().decompress(); + if key.is_none().into() { + return None; + } + + return Some(key.unwrap()); +} + +pub fn verenc_verify(proof: VerencProof) -> bool { + let blinding_key = point_from_bytes(proof.blinding_pubkey); + if blinding_key.is_none() { + return false; + } + + let statement = point_from_bytes(proof.statement); + if statement.is_none() { + return false; + } + + let encryption_key = point_from_bytes(proof.encryption_key); + if encryption_key.is_none() { + return false; + } + + let mut polycom: Vec = Vec::new(); + for p in proof.polycom { + let com = point_from_bytes(p); + if com.is_none() { + return false; + } + + polycom.push(com.unwrap()); + } + + let mut ctexts: Vec<(PKECipherText, usize)> = Vec::new(); + for c in proof.ctexts { + let c1 = point_from_bytes(c.c1); + if c1.is_none() { + return false; + } + + if c.c2.len() != 56 { + return false; + } + + let c2 = Scalar::from_bytes(&c.c2.try_into().unwrap()); + ctexts.push((PKECipherText{c1: c1.unwrap(), c2: c2}, c.i as usize)); + } + + let mut shares: Vec<(Scalar, Scalar, usize)> = Vec::new(); + for s in proof.shares_rands { + if s.s1.len() != 56 { + return false; + } + + if s.s2.len() != 56 { + return false; + } + + let s1 = Scalar::from_bytes(&s.s1.try_into().unwrap()); + let s2 = Scalar::from_bytes(&s.s2.try_into().unwrap()); + shares.push((s1, s2, s.i as usize)); + } + + let params = CurveParams::init(blinding_key.unwrap()); + let pke = Elgamal::setup(¶ms); + let (N, t, n) = RVE_PARAMS[0]; + let vparams = RDkgithParams{ N, t, n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + ve.set_ek(PKEPublicKey{ + ek: encryption_key.unwrap(), + }); + + return ve.verify(&statement.unwrap(), &RDkgithProof{ + challenge: proof.challenge, + polycom: polycom, + ctexts: ctexts, + shares_rands: shares, + }); +} + + +pub fn verenc_compress(proof: VerencProof) -> CompressedCiphertext { + let blinding_key = point_from_bytes(proof.blinding_pubkey); + if blinding_key.is_none() { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + let statement = point_from_bytes(proof.statement); + if statement.is_none() { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + let encryption_key = point_from_bytes(proof.encryption_key); + if encryption_key.is_none() { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + let mut polycom: Vec = Vec::new(); + for p in proof.polycom { + let com = point_from_bytes(p); + if com.is_none() { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + polycom.push(com.unwrap()); + } + + let mut ctexts: Vec<(PKECipherText, usize)> = Vec::new(); + for c in proof.ctexts { + let c1 = point_from_bytes(c.c1); + if c1.is_none() { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + if c.c2.len() != 56 { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + let c2 = Scalar::from_bytes(&c.c2.try_into().unwrap()); + ctexts.push((PKECipherText{c1: c1.unwrap(), c2: c2}, c.i as usize)); + } + + let mut shares: Vec<(Scalar, Scalar, usize)> = Vec::new(); + for s in proof.shares_rands { + if s.s1.len() != 56 { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + if s.s2.len() != 56 { + return CompressedCiphertext{ + ctexts: vec![], + aux: vec![], + }; + } + + let s1 = Scalar::from_bytes(&s.s1.try_into().unwrap()); + let s2 = Scalar::from_bytes(&s.s2.try_into().unwrap()); + shares.push((s1, s2, s.i as usize)); + } + + let params = CurveParams::init(blinding_key.unwrap()); + let pke = Elgamal::setup(¶ms); + let (N, t, n) = RVE_PARAMS[0]; + let vparams = RDkgithParams{ N, t, n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + ve.set_ek(PKEPublicKey{ + ek: encryption_key.unwrap(), + }); + let ve_ct = ve.compress(&statement.unwrap(), &RDkgithProof{ + challenge: proof.challenge, + polycom: polycom, + ctexts: ctexts, + shares_rands: shares, + }); + return CompressedCiphertext{ + ctexts: ve_ct.ctexts.iter().map(|v| VerencCiphertext{ + c1: v.c1.compress().to_bytes().to_vec(), + c2: v.c2.to_bytes().to_vec(), + i: 0, + }).collect(), + aux: ve_ct.aux.iter().map(|a| a.to_bytes().to_vec()).collect(), + }; +} + +pub fn verenc_recover(recovery: VerencDecrypt) -> Vec { + let blinding_key = point_from_bytes(recovery.blinding_pubkey); + if blinding_key.is_none() { + return vec![]; + } + + let statement = point_from_bytes(recovery.statement); + if statement.is_none() { + return vec![]; + } + + if recovery.decryption_key.len() != 56 { + return vec![]; + } + + let decryption_key = Scalar::from_bytes(&recovery.decryption_key.try_into().unwrap()); + let mut ctexts: Vec = Vec::new(); + let mut aux: Vec = Vec::new(); + for c in recovery.ciphertexts.ctexts { + let c1 = point_from_bytes(c.c1); + if c1.is_none() { + return vec![]; + } + + if c.c2.len() != 56 { + return vec![]; + } + + let c2 = Scalar::from_bytes(&c.c2.try_into().unwrap()); + ctexts.push(PKECipherText{c1: c1.unwrap(), c2: c2}); + } + + for c in recovery.ciphertexts.aux { + if c.len() != 56 { + return vec![]; + } + + let a = Scalar::from_bytes(&c.try_into().unwrap()); + aux.push(a); + } + + let ve_ct = RDkgithCipherText{ + ctexts: ctexts, + aux: aux, + }; + + let params = CurveParams::init(blinding_key.unwrap()); + let pke = Elgamal::setup(¶ms); + let (N, t, n) = RVE_PARAMS[0]; + let vparams = RDkgithParams{ N, t, n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + let wit_recover = ve.recover(&statement.unwrap(), &decryption_key, &ve_ct); + return wit_recover.to_bytes().to_vec(); +} + +pub fn chunk_data_for_verenc(data: Vec) -> Vec> { + return encode_to_curve448_scalars(&data); +} + +pub fn combine_chunked_data(chunks: Vec>) -> Vec { + return decode_from_curve448_scalars(&chunks); +} + +#[cfg(test)] +mod tests { + use rand::RngCore; + + use super::*; + + #[test] + fn test_verenc() { + let data = vec![0, 'h' as u8, 'e' as u8, 'l' as u8, 'l' as u8, 'o' as u8, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0]; + let proof = new_verenc_proof(data.clone()); + let proofdata = proof.clone(); + let pubproof = VerencProof { blinding_pubkey: proof.blinding_pubkey, encryption_key: proof.encryption_key, statement: proof.statement, challenge: proof.challenge, polycom: proof.polycom, ctexts: proof.ctexts, shares_rands: proof.shares_rands }; + assert!(verenc_verify(pubproof.clone())); + let compressed = verenc_compress(pubproof); + let result = verenc_recover(VerencDecrypt{ + blinding_pubkey: proofdata.blinding_pubkey, + decryption_key: proof.decryption_key, + statement: proofdata.statement, + ciphertexts: compressed, + }); + + assert!(data == result); + } + + #[test] + fn test_chunking() { + for i in 0..1000 { + let mut data: [u8; 1300] = [0u8;1300]; + OsRng::fill_bytes(&mut OsRng, &mut data); + let chunks = chunk_data_for_verenc(data.to_vec()); + for chunk in chunks.clone() { + let scalar_chunk = Scalar::from_bytes(&chunk.clone().try_into().unwrap()); + assert!(scalar_chunk.to_bytes().to_vec() == chunk) + } + let result = combine_chunked_data(chunks); + let mut padded_data = data.to_vec(); + while result.len() > padded_data.len() { + padded_data.push(0); + } + assert!(padded_data == result); + } + } + + #[test] + fn test_full_verenc() { + let mut data: [u8; 128] = [0u8;128]; + OsRng::fill_bytes(&mut OsRng, &mut data); + let chunks = chunk_data_for_verenc(data.to_vec()); + for chunk in chunks.clone() { + let proof = new_verenc_proof(chunk.clone()); + let proofdata = proof.clone(); + let pubproof = VerencProof { blinding_pubkey: proof.blinding_pubkey, encryption_key: proof.encryption_key, statement: proof.statement, challenge: proof.challenge, polycom: proof.polycom, ctexts: proof.ctexts, shares_rands: proof.shares_rands }; + assert!(verenc_verify(pubproof.clone())); + let compressed = verenc_compress(pubproof); + let result = verenc_recover(VerencDecrypt{ + blinding_pubkey: proofdata.blinding_pubkey, + decryption_key: proof.decryption_key, + statement: proofdata.statement, + ciphertexts: compressed, + }); + + assert!(chunk == result); + } + let result = combine_chunked_data(chunks); + let mut padded_data = data.to_vec(); + while result.len() > padded_data.len() { + padded_data.push(0); + } + assert!(padded_data != result); + } +} \ No newline at end of file diff --git a/crates/verenc/src/lib.udl b/crates/verenc/src/lib.udl new file mode 100644 index 0000000..a636d1b --- /dev/null +++ b/crates/verenc/src/lib.udl @@ -0,0 +1,55 @@ +namespace verenc { + VerencProofAndBlindingKey new_verenc_proof(sequence data); + VerencProofAndBlindingKey new_verenc_proof_encrypt_only(sequence data, sequence encryption_key_bytes); + boolean verenc_verify(VerencProof proof); + CompressedCiphertext verenc_compress(VerencProof proof); + sequence verenc_recover(VerencDecrypt recovery); + sequence> chunk_data_for_verenc(sequence data); + sequence combine_chunked_data(sequence> chunks); +}; + +dictionary VerencCiphertext { + sequence c1; + sequence c2; + u64 i; +}; + +dictionary VerencShare { + sequence s1; + sequence s2; + u64 i; +}; + +dictionary VerencProofAndBlindingKey { + sequence blinding_key; + sequence blinding_pubkey; + sequence decryption_key; + sequence encryption_key; + sequence statement; + sequence challenge; + sequence> polycom; + sequence ctexts; + sequence shares_rands; +}; + +dictionary VerencDecrypt { + sequence blinding_pubkey; + sequence decryption_key; + sequence statement; + CompressedCiphertext ciphertexts; +}; + +dictionary VerencProof { + sequence blinding_pubkey; + sequence encryption_key; + sequence statement; + sequence challenge; + sequence> polycom; + sequence ctexts; + sequence shares_rands; +}; + +dictionary CompressedCiphertext { + sequence ctexts; + sequence> aux; +}; diff --git a/crates/verenc/src/pke.rs b/crates/verenc/src/pke.rs new file mode 100644 index 0000000..823aec0 --- /dev/null +++ b/crates/verenc/src/pke.rs @@ -0,0 +1,116 @@ +/* Hashed Elgamal implementation */ +#![allow(dead_code)] +#![allow(non_snake_case)] + +use crate::utils::*; + +use ed448_goldilocks_plus::elliptic_curve::Group; +use rand::rngs::OsRng; +use ed448_goldilocks_plus::EdwardsPoint as GGA; +use ed448_goldilocks_plus::Scalar as FF; + +const WINDOW_SIZE : usize = 7; + +#[derive(Clone)] +pub struct Elgamal { + pub(crate) params: CurveParams, + pub(crate) G : GGA +} + +#[derive(Copy, Clone, Default, Debug)] +pub struct PKECipherText { + pub(crate) c1 : GGA, + pub(crate) c2 : FF, +} + +#[derive(Clone)] +pub struct PKEPublicKey { + pub(crate) ek : GGA, +} + +impl PKECipherText { + pub fn zero() -> Self { + PKECipherText {c1: GGA::IDENTITY, c2: FF::ZERO} + } +} + +impl PKECipherText { + pub fn to_bytes(&self) -> Vec { + let c1_bytes = self.c1.compress().to_bytes().to_vec(); + let c2_bytes = self.c2.to_bytes().to_vec(); + [c1_bytes, c2_bytes].concat() + } +} + + +impl Elgamal { + pub fn setup(params: &CurveParams) -> Self { + // we skip a lot of precomputation utilizing edwards curves + let precomp_G = GGA::generator(); + + Elgamal { params: params.clone(), G: precomp_G } + } + + pub fn kgen(&self) -> (PKEPublicKey, FF) { + let x = FF::random(&mut OsRng); + let Y = self.mul_G(x); + + let pk = PKEPublicKey{ek: Y}; + + return (pk, x); + } + + pub fn encrypt(&self, ek: &PKEPublicKey, msg: &FF) -> PKECipherText { + self.encrypt_given_r(ek, msg, &FF::random(&mut OsRng)) + } + + fn mul_G(&self, scalar : FF) -> GGA { + self.G * scalar + } + fn mul_ek(ek : &GGA, scalar : FF) -> GGA { + ek * scalar + } + + pub fn encrypt_given_r(&self, ek: &PKEPublicKey, msg: &FF, r: &FF) -> PKECipherText { + let c1 = self.mul_G(*r); + self.encrypt_given_c1(ek, msg, r, c1) + } + + // Encryption where c1 = G^r is given + pub fn encrypt_given_c1(&self, ek: &PKEPublicKey, msg: &FF, r: &FF, c1 : GGA) -> PKECipherText { + let keyseed = Self::mul_ek(&ek.ek, *r); + let hash = hash_to_FF(&keyseed); + let c2 = hash + msg; + PKECipherText { c1, c2 } + } + + pub fn decrypt(&self, dk: &FF, ct: &PKECipherText) -> FF { + let pt = ct.c1 * dk; + let hash = hash_to_FF(&pt); + ct.c2 - hash + } + +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_pke_kgen() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + let (ek, dk) = pke.kgen(); + assert_eq!(params.G * dk, ek.ek); + + } + + #[test] + fn test_pke_enc_dec() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + let (ek, dk) = pke.kgen(); + let m = FF::random(&mut OsRng); + let ct = pke.encrypt(&ek, &m); + let pt = pke.decrypt(&dk, &ct); + assert_eq!(m, pt); + } +} \ No newline at end of file diff --git a/crates/verenc/src/rdkgith.rs b/crates/verenc/src/rdkgith.rs new file mode 100644 index 0000000..d80bc2b --- /dev/null +++ b/crates/verenc/src/rdkgith.rs @@ -0,0 +1,531 @@ +#![allow(non_snake_case)] + +use crate::utils::*; +use crate::pke::*; +use crate::ve::*; + +use rand::rngs::OsRng; +use rand::seq::IteratorRandom; +use sha2::{Digest, Sha512}; + + +use ed448_goldilocks_plus::elliptic_curve::{Group, PrimeField}; +use ed448_goldilocks_plus::EdwardsPoint as GGA; +use ed448_goldilocks_plus::Scalar as FF; + +pub const RVE_PARAMS : [(usize, usize, usize); 1] = [(64, 22, 3)]; + +pub const WINDOW_SIZE : usize = 7; +pub const FIELD_ELT_BYTES : usize = ((FF::NUM_BITS + 7) / 8) as usize; + +#[derive(Clone)] +pub struct RDkgithParams { + pub N: usize, // number of parties + pub t: usize, // number of parallel repetitions + pub n: usize, // size of random subset +} + +#[derive(Clone, Debug)] +pub struct RDkgithProof { + pub(crate) challenge : Vec, + pub(crate) polycom: Vec, // A_1,..., A_t + pub(crate) ctexts : Vec<(PKECipherText, usize)>, // unopened ciphertexts ct_i + //pub(crate) shares: Vec>, // opened (s_i)_{i\in I} + //pub(crate) rands: Vec>, // opened (r_i)_{i\in I} + pub(crate) shares_rands: Vec<(FF, FF, usize)>, +} + +#[derive(Clone, Debug)] +pub struct RDkgithCipherText { + pub(crate) ctexts : Vec, + pub(crate) aux: Vec +} + +#[derive(Clone)] +pub struct RDkgith { + pub(crate) params: CurveParams, + pub(crate) vparams: RDkgithParams, + pub(crate) pke: Elgamal, + pub(crate) ek: PKEPublicKey, + pub(crate) precomp_G : GGA +} + +impl RDkgith { + pub fn check_instance(&self, stm: &GGA, wit: &FF) -> bool { + if &(self.params.G * wit) == stm { + return true + } + false + } + + pub fn expand_challenge(&self, challenge: &Vec) -> Vec { + let length_required = self.vparams.N - self.vparams.t; + let mut output = Vec::::new(); + let mut c = challenge.clone(); + while output.len() < length_required { + + let ints = bytes_to_u32(&c); + for i in 0..ints.len() { + let idx = (ints[i] as usize) % self.vparams.N; + if !output.contains(&idx) { + output.push(idx); + } + if output.len() == length_required { + break; + } + } + + if output.len() != length_required { + c = hash_SHA512(c.as_slice()); + } + } + output.sort(); + output + } + + fn mul_G(&self, scalar : FF) -> GGA { + self.precomp_G * scalar + } +} + +impl VerEnc for RDkgith { + type SystemParams = CurveParams; + type Statement = GGA; + type Witness = FF; + type PKE = Elgamal; + type EncKey = PKEPublicKey; + type DecKey = FF; + type VEParams = RDkgithParams; + type VEProof = RDkgithProof; + type VECipherText = RDkgithCipherText; + + fn setup(params: &CurveParams, vparams: &Self::VEParams, pke: Self::PKE) -> Self { + let precomp_G = GGA::generator(); + RDkgith { params: params.clone(), vparams: vparams.clone(), pke, + ek : PKEPublicKey{ek: GGA::identity()}, + precomp_G + } + } + + fn set_ek(&mut self, ek: PKEPublicKey) { + self.ek = ek; + } + + fn kgen(&mut self) -> Self::DecKey { + let (ek, dk) = self.pke.kgen(); + self.ek = ek; + return dk; + } + + fn get_public_key(&self) -> &Self::EncKey { + &self.ek + } + + fn igen(&self, w: &[u8;56]) -> (Self::Statement, Self::Witness) { + let x = FF::from_bytes(w); + let Y = if self.params.G == GGA::generator() { + self.params.G * x + } else { + self.params.G * x + }; + return (Y, x); + } + + fn prove(&self, stm: &Self::Statement, wit: &Self::Witness) -> Self::VEProof { + let N = self.vparams.N; + let t = self.vparams.t; + let mut hasher = Sha512::new(); + + + let mut coeffs = Vec::::with_capacity(t+1); + let mut polycom = Vec::::with_capacity(t+1); + + let mut ctexts = Vec::::with_capacity(N); + let mut shares = Vec::::with_capacity(N); + let mut rands = Vec::::with_capacity(N); + let mut ret_ctexts = Vec::<(PKECipherText, usize)>::with_capacity(N-t); + let mut ret_shares_rands = Vec::<(FF, FF, usize)>::with_capacity(t); + + + + /* Sample and commit to polynomial */ + for j in 0..t+1 { + let aj = + if j == 0 { + wit.clone() + } else { + FF::random(&mut OsRng) + }; + let Aj = self.mul_G(aj); + + coeffs.insert(j, aj); + polycom.insert(j, Aj); + + // hash + let Aj_bytes = Aj.compress().to_bytes().to_vec(); + hasher.update(Aj_bytes); + } + + for i in 0..N { + let mut s = coeffs[0]; + let x = FF::from((i+1) as u32); + let mut xi = x; + + for j in 1..coeffs.len() { + let term = coeffs[j] * xi; + xi *= x; + s += term; + } + let r = FF::random(&mut OsRng); + let ct = self.pke.encrypt_given_r(&self.ek, &s, &r); + shares.insert(i, s); + rands.insert(i, r); + ctexts.insert(i, ct); + + // hash + hasher.update(ct.to_bytes()); + } + + // Hash stm and ek + let stm_bytes = stm.compress().to_bytes().to_vec(); + let ek_bytes = self.ek.ek.compress().to_bytes().to_vec(); + hasher.update(stm_bytes); + hasher.update(ek_bytes); + + let chal = hasher.finalize().to_vec(); + let p_indices = self.expand_challenge(&chal); + + // construct proof + for i in 0..N { + if p_indices.contains(&i) { + ret_ctexts.push((ctexts[i], i)); + } else { + ret_shares_rands.push((shares[i], rands[i], i)); + } + } + + RDkgithProof { + challenge: chal, + polycom, + ctexts: ret_ctexts, + shares_rands: ret_shares_rands + } + } + + fn verify(&self, stm: &Self::Statement, pi: &Self::VEProof) -> bool { + let N = self.vparams.N; + let t = self.vparams.t; + let mut hasher = Sha512::new(); + + + // index of hidden parties + let p_indices = self.expand_challenge(&pi.challenge); + + // hash polycom + for j in 0..t+1 { + let Aj = pi.polycom[j]; + let Aj_bytes = Aj.compress().to_bytes().to_vec(); + hasher.update(Aj_bytes); + } + + // check input format + if pi.ctexts.len() != N-t || pi.shares_rands.len() != t || p_indices.len() != N-t { + return false; + } + // Reconstruct missing ciphertexts + let mut ctr_hide = 0; + let mut ctr_open = 0; + for i in 0..N { + if p_indices.contains(&i) { + let (ct, idx) = pi.ctexts[ctr_hide]; + hasher.update(ct.to_bytes()); + if i != idx { + return false; + } + ctr_hide += 1; + + } else { + let (s, r, idx) = pi.shares_rands[ctr_open]; + let ct = self.pke.encrypt_given_r(&self.ek, &s, &r); + hasher.update(ct.to_bytes()); + if i != idx { + return false; + } + ctr_open += 1; + } + } + // Hash stm and ek + let stm_bytes = stm.compress().to_bytes().to_vec(); + let ek_bytes = self.ek.ek.compress().to_bytes().to_vec(); + hasher.update(stm_bytes); + hasher.update(ek_bytes); + + // check hash + let chal_rec = hasher.finalize().to_vec(); + if chal_rec != pi.challenge { + return false; + } + + // Check shares -- Batched implementation: requires computing 1 MSM with t+1 terms + // See the "small exponents test" from the paper: + // Fast batch verification for modular exponentiation and digital signatures. Mihir Bellare, Juan A. Garay & Tal Rabin, EUROCRYPT'98 + // Basically the verifier takes a random linear combination of the LHSs and RHSs + let mut left_scalar = FF::ZERO; + let mut right_scalars = vec![FF::ZERO; t+1]; + + for (s, _, i) in &pi.shares_rands { + let random_d = FF::random(&mut OsRng); + // Compute scalars for RHS + let i_FF = FF::from(*i as u32 + 1); + let mut i_pow = FF::from(1 as u32); + for j in 0..t+1 { + right_scalars[j] += i_pow * random_d; + i_pow = i_pow * i_FF; + } + left_scalar += s * &random_d; + + } + let left = self.mul_G(left_scalar); + let mut right = GGA::identity(); + for i in 0..pi.polycom.len() { + right += pi.polycom[i] * &right_scalars[i] + } + if left != right { + return false; + } + + true + } + + // Lagrange coeff: product delta_i(0) = prod_{j\neq i} j/(j-i) + // Postprocessed ciphertext for party index i^*: + // c1 = r * G + // c2 = delta_{i^*}(0) (H(r * ek) + s_{i^*}) + sum_{i\neq i^*} delta_{i}(0) s_i + fn compress(&self, _stm: &Self::Statement, pi: &Self::VEProof) -> Self::VECipherText { + let N = self.vparams.N; + let t = self.vparams.t; + let n = self.vparams.n; + let mut new_ctexts = Vec::::with_capacity(n); + let mut aux = Vec::::with_capacity(n); + let hide_indices = self.expand_challenge(&pi.challenge); + let mut open_indices = Vec::::with_capacity(t); + + let mut lagrange = vec![FF::ZERO; N]; + for i in 0..N { + if !hide_indices.contains(&i) { + open_indices.push(i); + } + } + + assert_eq!(open_indices.len(), t); + + // preprocess lagrange + for i in open_indices.iter() { + let i_FF = FF::from(*i as u32 + 1); + let mut prod = FF::from(1 as u32); + let mut denom = FF::from(1 as u32); + for j in open_indices.iter() { + if j != i { + let j_FF = FF::from(*j as u32 + 1); + prod = prod * j_FF; + denom = denom * (j_FF - i_FF); + } + } + lagrange[*i] = prod * denom.invert(); + } + + // sample random subset of size n + let subset= hide_indices.iter().choose_multiple(&mut OsRng, n); + + let mut ctr_hide = 0; + // process each ciphertext + for i_hide in hide_indices.iter() { + if !subset.contains(&i_hide) { + ctr_hide += 1; + continue; + } + + let (ct, _idx) = pi.ctexts[ctr_hide]; + let c1_new = ct.c1; + let mut c2_new = ct.c2; + let i_hide_FF = FF::from(*i_hide as u32 + 1); + let mut prod = FF::from(1 as u32); + + // multiply c2 by i_hide's lagrange + for j in open_indices.iter() { + if j != i_hide { + let j_FF = FF::from(*j as u32 + 1); + prod = (prod * j_FF) * (j_FF - i_hide_FF).invert(); + } + } + c2_new = c2_new * prod; + + // add sum of lagrange * s_i to c2 + let mut ctr_open = 0; + for i in open_indices.iter() { + let i_FF = FF::from(*i as u32 + 1); + let mut delta_i = lagrange[*i]; + delta_i = (delta_i * i_hide_FF) * (i_hide_FF - i_FF).invert(); // update delta_i using i_hide + let (s,_,_) = pi.shares_rands[ctr_open]; + c2_new = c2_new + delta_i * s; + ctr_open += 1; + } + + new_ctexts.push(PKECipherText { c1: c1_new, c2: c2_new }); + aux.push(prod); + + ctr_hide += 1; + + } + + RDkgithCipherText { + ctexts: new_ctexts, + aux // TODO: maybe receiver can recompute this from party indices + } + } + + fn recover(&self, stm: &Self::Statement, dk: &Self::DecKey, ve_ct: &Self::VECipherText) -> Self::Witness { + let n = self.vparams.n; + for i in 0..n { + let ct = ve_ct.ctexts[i]; + let delta = ve_ct.aux[i]; + let pt = ct.c1 * dk; + let hash = hash_to_FF(&pt); + let ptext = ct.c2 - hash * delta; + if self.check_instance(stm, &ptext) { + return ptext; + } + } + print!("recovery failed!"); + FF::ZERO + } + +} + +#[cfg(test)] +mod tests { + use std::time::Instant; + + use ed448_goldilocks_plus::Scalar; + + use super::*; + + #[test] + fn test_ve_kgen() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + let vparams = RDkgithParams{ N: 8, t: 4, n: 4}; + let mut ve = RDkgith::setup(¶ms, &vparams, pke); + let dk = ve.kgen(); + + assert_eq!(params.G * dk, ve.get_public_key().ek); + assert_eq!(params.G * dk, ve.get_public_key().ek); + } + + #[test] + fn test_ve_igen() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + let vparams = RDkgithParams{ N: 8, t: 4, n: 4}; + let ve = RDkgith::setup(¶ms, &vparams, pke); + let w = Scalar::random(&mut OsRng); + let (stm, wit) = ve.igen(&w.to_bytes()); + assert_eq!(params.G * wit, stm) + } + + #[test] + fn test_ve_prove_verify() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + for (N, t, n) in &RVE_PARAMS[0..1] { + let vparams = RDkgithParams{ N: *N, t: *t, n: *n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + let _dk = ve.kgen(); + let w = Scalar::random(&mut OsRng); + let (stm, wit) = ve.igen(&w.to_bytes()); + let pi = ve.prove(&stm, &wit); + println!("proof generated"); + let result = ve.verify(&stm, &pi); + println!("proof verified"); + assert!(result); + } + } + + #[test] + fn test_ve_prove_compress_recover() { + let params = CurveParams::init(GGA::generator()); + let pke = Elgamal::setup(¶ms); + for (N, t, n) in &RVE_PARAMS[0..1] { + let vparams = RDkgithParams{ N: *N, t: *t, n: *n }; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + let dk = ve.kgen(); + let w = Scalar::random(&mut OsRng); + let (stm, wit) = ve.igen(&w.to_bytes()); + let pi = ve.prove(&stm, &wit); + println!("proof generated"); + let ve_ct = ve.compress(&stm, &pi); + println!("VE ciphertext generated"); + let wit_recover = ve.recover(&stm, &dk, &ve_ct); + assert_eq!(wit_recover, wit); + } + } + + + pub fn proof_size(pi : &RDkgithProof) -> usize { + let group_elt_bytes = 57; + + let mut size = pi.challenge.len(); + size += pi.polycom.len() * group_elt_bytes; + size += pi.ctexts.len() * (pke_ctext_size(&pi.ctexts[0].0) + 8); + size += pi.shares_rands.len() * (2*FIELD_ELT_BYTES + 8); + + size + } + pub fn ctext_size(ctext : &RDkgithCipherText) -> usize { + let mut size = ctext.ctexts.len() * pke_ctext_size(&ctext.ctexts[0]); + size += ctext.aux.len() * FIELD_ELT_BYTES; + + size + } + pub fn pke_ctext_size(_ctext : &PKECipherText) -> usize { + let group_elt_bytes = 57; + let size = group_elt_bytes + FIELD_ELT_BYTES; + + size + + } + + #[test] + fn test_ve_print_sizes() { + let blind = GGA::random(&mut OsRng); + let params = CurveParams::init(blind); + let pke = Elgamal::setup(¶ms); + + for (N, t, n) in RVE_PARAMS { + let vparams = RDkgithParams{ N, t, n}; + let mut ve = RDkgith::setup(¶ms, &vparams, pke.clone()); + let dk = ve.kgen(); + let w = Scalar::random(&mut OsRng); + let (stm, wit) = ve.igen(&w.to_bytes()); + let start = Instant::now(); + let pi = ve.prove(&stm, &wit); + let duration = start.elapsed(); + print!("\nN = {}, t = {}, n = {}\n", N, t, n); + print!("Proof size : {}, duration : {:?}\n", proof_size(&pi), duration); + let start = Instant::now(); + assert!(ve.verify(&stm, &pi)); + let duration = start.elapsed(); + print!("verification duration : {:?}\n", duration); + let start = Instant::now(); + let ve_ct = ve.compress(&stm, &pi); + let duration = start.elapsed(); + print!("Ctext size : {}\n", (N-t) * (pke_ctext_size(&ve_ct.ctexts[0]) + FIELD_ELT_BYTES)); + print!("Ctext size (RS): {}, compression duration : {:?}\n", ctext_size(&ve_ct), duration); + let wit_recover = ve.recover(&stm, &dk, &ve_ct); + + assert_eq!(wit_recover, wit); + } + } + +} + diff --git a/crates/verenc/src/seed_tree.rs b/crates/verenc/src/seed_tree.rs new file mode 100644 index 0000000..33279a3 --- /dev/null +++ b/crates/verenc/src/seed_tree.rs @@ -0,0 +1,199 @@ +#![allow(non_snake_case)] + +use std::convert::TryInto; +use sha2::{Digest, Sha256}; +use rand::RngCore; +use rand::rngs::OsRng; + +// Implementation of the seed tree optimization +// Port of the LegRoast C implementation to Rust +// https://github.com/WardBeullens/LegRoast, main branch at cac7406) +// See merkletree.c + +// To convert seeds to finite field elements see utils::seed_to_FF + +pub const SEED_BYTES : usize = 16; +pub type Seed = [u8; SEED_BYTES]; + +pub struct SeedTree { + seeds : Vec, // length must be (2*PARTIES-1) + depth : usize, // log_2(N) + num_leaves: usize // N +} + +impl SeedTree { + + fn expand(salt : &[u8], rep_index : u16, seed_index : u16, seed : &Seed) -> (Seed, Seed) { + let mut hasher = Sha256::new(); + hasher.update(salt); + hasher.update(rep_index.to_le_bytes()); + hasher.update(seed_index.to_le_bytes()); + hasher.update(seed); + let digest = hasher.finalize(); + + ( digest[0..SEED_BYTES].try_into().expect("Hash digest too short, needs to be twice the seed length"), + digest[SEED_BYTES..].try_into().expect("Hash digest too short, needs to be twice the seed length") ) + } + + fn left_child(i : usize) -> usize { + 2*i+1 + } + fn right_child(i : usize) -> usize { + 2*i+2 + } + fn parent(i : usize) -> usize { + (i-1)/2 + } + fn sibling(i : usize) -> usize { + // ((i)%2)? i+1 : i-1 + if i % 2 == 1 { + i + 1 + } else { + i - 1 + } + } + + pub fn zero_seed() -> Seed { + [0; SEED_BYTES] + } + + pub fn random_seed() -> Seed { + let mut random_vector = [0u8; SEED_BYTES]; + OsRng.fill_bytes(&mut random_vector); + random_vector + } + + pub fn create(root_seed : &Seed, depth : usize, salt : &[u8], rep_index : usize) -> Self { + + let num_leaves = 1 << depth; + let mut seeds = vec![Self::zero_seed(); 2*num_leaves - 1]; + seeds[0] = root_seed.clone(); + let rep_index = rep_index as u16; + + for i in 0 .. num_leaves - 1 { + let i_u16 : u16 = i.try_into().unwrap(); + let (left, right) = Self::expand(salt, rep_index, i_u16, &seeds[i]); + seeds[Self::left_child(i)] = left; + seeds[Self::right_child(i)] = right; + } + + SeedTree{seeds, depth, num_leaves} + } + + // Unopened party index is given in [0, .., N-1] + pub fn open_seeds(&self, unopened_index : usize) -> Vec { + let mut unopened_index = unopened_index + (1 << self.depth) - 1; + let mut out = Vec::new(); + let mut to_reveal = 0; + while to_reveal < self.depth { + out.push(self.seeds[Self::sibling(unopened_index)]); + unopened_index = Self::parent(unopened_index); + to_reveal += 1; + } + + out + } + + // Callers must ensure that revealed.size() == depth + pub fn reconstruct_tree(depth : usize, salt : &[u8], rep_index : usize, unopened_index : usize, revealed : &Vec) -> Self { + let num_leaves = 1 << depth; + let mut unopened_index = unopened_index + num_leaves - 1; + let mut seeds = vec![Self::zero_seed(); 2 * num_leaves - 1]; + let mut next_insert = 0; + assert!(revealed.len() == depth); + while next_insert < depth { + seeds[Self::sibling(unopened_index)] = revealed[next_insert]; + unopened_index = Self::parent(unopened_index); + next_insert += 1; + } + + let zero_seed = seeds[0]; // we'll never have the root + for i in 0 .. num_leaves - 1 { + if seeds[i] != zero_seed { + let (left, right) = Self::expand(salt, rep_index as u16, i as u16, &seeds[i]); + seeds[Self::left_child(i)] = left; + seeds[Self::right_child(i)] = right; + } + } + + SeedTree { seeds, depth, num_leaves } + } + + pub fn get_leaf(&self, i : usize) -> Seed { + assert!(i < self.num_leaves, "get_leaf: leaf index too large"); // Caller bug + + self.seeds[self.num_leaves - 1 + i] + } + + pub fn print_tree(&self, label : &str) { + print!("Tree {}:\n", label); + for i in 0..self.seeds.len() { + print!("seed {} = {}\n", i, hex::encode_upper(self.seeds[i])); + if i == self.num_leaves - 2 { + print!("---- leaves follow ----\n") + } + } + } + +} + + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::OsRng; + use rand::RngCore; + + fn random_vec(len :usize) -> Vec { + let mut random_vector = vec![0u8; len]; + OsRng.fill_bytes(&mut random_vector); + random_vector + } + + + + #[test] + fn test_seed_tree_create() { + let N = 8; + let logN = 3; + let root_seed = SeedTree::random_seed(); + let salt = random_vec(32); + let rep_index = 5; + + let tree = SeedTree::create(&root_seed, logN, salt.as_slice(), rep_index); + assert!(tree.num_leaves == N); + for i in 0..tree.num_leaves { + let leaf_seed_i = tree.get_leaf(i); + assert!(leaf_seed_i != SeedTree::zero_seed()); + } + } + + #[test] + fn test_seed_tree_roundtrip() { + let N = 8; + let logN = 3; + let root_seed = SeedTree::random_seed(); + let salt = random_vec(32); + let rep_index = 5; + + let tree = SeedTree::create(&root_seed, logN, salt.as_slice(), rep_index); + assert!(tree.num_leaves == N); + + for unopened_party in 0 .. N-1 { + let opening_data = tree.open_seeds(unopened_party); + let tree2 = SeedTree::reconstruct_tree(logN, &salt, rep_index, unopened_party, &opening_data); + assert!(tree2.num_leaves == N); + + for i in 0..N { + if i != unopened_party { + assert!(tree.get_leaf(i) == tree2.get_leaf(i)); + } + else { + assert!(tree2.get_leaf(i) == SeedTree::zero_seed()); + } + } + } + + } + +} \ No newline at end of file diff --git a/crates/verenc/src/utils.rs b/crates/verenc/src/utils.rs new file mode 100644 index 0000000..1682802 --- /dev/null +++ b/crates/verenc/src/utils.rs @@ -0,0 +1,189 @@ +#![allow(non_snake_case)] + +use sha2::{Digest, Sha512}; +use crate::Seed; +use std::convert::TryInto; + +use ed448_goldilocks_plus::EdwardsPoint as GGA; +use ed448_goldilocks_plus::Scalar as FF; + + +pub type Statement = GGA; +pub type Witness = FF; + + +#[derive(Clone, Default, PartialEq, Debug)] +pub struct CurveParams { + pub(crate) G: GGA, +} + +impl CurveParams { + pub fn init(blind: GGA) -> Self { + let G = blind; + + CurveParams { + G + } + } +} + +/* Utility functions */ +pub fn hash_to_FF(point: &GGA) -> FF { + let digest = hash_SHA512(&point.compress().to_bytes()[..]); + + FF::from_bytes(&(digest[0..56].try_into().unwrap())) +} + +pub fn hash_SHA512(input : &[u8]) -> Vec { + let mut hasher = Sha512::new(); + hasher.update(input); + + hasher.finalize().to_vec() +} + +pub fn bytes_to_u32(input : &Vec) -> Vec { + let extra = input.len() % 4; + let mut output = Vec::::new(); + for i in (0..input.len()-extra).step_by(4) { + let next_bytes : [u8 ; 4] = input[i..i+4].try_into().unwrap(); + output.push(u32::from_le_bytes(next_bytes)); + } + output +} + +// Derive a uniformly random field element from a seed, +// assuming the bitlength of the field is less than 448 bits +// Used to convert seeds to shares. +pub fn seed_to_FF(seed: Seed, salt: &[u8], rep_index : usize, party_index : usize, additional_input : Option<&[u8]>) -> FF { + let rep_index = rep_index as u16; + let party_index = party_index as u16; + let mut hasher = Sha512::new(); + hasher.update(salt); + hasher.update(seed); + hasher.update(rep_index.to_le_bytes()); + hasher.update(party_index.to_le_bytes()); + if additional_input.is_some() { + hasher.update(additional_input.unwrap()); + } + + let digest = hasher.finalize(); + + FF::from_bytes(&digest[0..56].try_into().unwrap()) +} + +// A simple bit–reader over a byte slice. +struct BitReader<'a> { + data: &'a [u8], + // This counts the total number of bits read so far. + bit_pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, bit_pos: 0 } + } + + // Reads a single bit from the input. + fn read_bit(&mut self) -> Option { + if self.bit_pos >= self.data.len() * 8 { + return None; + } + // In little–endian order within a byte, the bit at position (bit_pos % 8) + // is extracted. (This is just one valid choice; what matters is consistency.) + let byte = self.data[self.bit_pos / 8]; + let bit = (byte >> (self.bit_pos % 8)) & 1; + self.bit_pos += 1; + Some(bit != 0) + } + + // Reads `count` bits and returns them as the lower `count` bits of a u64. + // (Assumes count <= 64.) + fn read_bits(&mut self, count: usize) -> Option { + let mut value = 0u64; + for i in 0..count { + // Each bit read is placed in position i (i.e. we’re building a little–endian number). + let bit = self.read_bit(); + if bit.is_some() && bit.unwrap() { + value |= 1 << i; + } + if i == 0 && bit.is_none() { + return None + } + } + Some(value) + } +} + +// Encodes an arbitrary byte slice into a vector of Curve448 clamped scalars. +// Each scalar encodes 432 bits of data (i.e. about 54 bytes). +// +// The mapping is as follows (little–endian view): +// - Byte 0: empty +// - Bytes 1..54: all 8 bits are free +// - Byte 55: empty +// +// If the final chunk has fewer than 432 bits, it is padded with zero bits. +pub fn encode_to_curve448_scalars(input: &[u8]) -> Vec> { + let mut reader = BitReader::new(input); + let mut scalars = Vec::new(); + + // Continue until no more bits are available. + while let Some(_) = reader.read_bits(1) { + // (We already advanced one bit; move back one step.) + reader.bit_pos -= 1; + + let mut scalar = vec![0;56]; + + for i in 1..55 { + // If there aren’t enough bits, pad with 0. + let byte = reader.read_bits(8).unwrap_or(0); + scalar[i] = byte as u8; + } + + scalars.push(scalar); + } + + scalars +} + +// Recombines a slice of 56-byte scalars (each containing 432 free bits) +// into the original bit–stream, returned as a Vec. +// +// The packing was as follows (little–endian bit–order): +// - Byte 0: empty +// - Bytes 1..54: all 8 bits are free (432 bits total) +// - Byte 55: empty +pub fn decode_from_curve448_scalars(scalars: &Vec>) -> Vec { + let mut output: Vec = Vec::new(); + // We'll accumulate bits in `acc` (lowest-order bits are the oldest) + // and keep track of how many bits we have in `bits_in_acc`. + let mut acc: u64 = 0; + let mut bits_in_acc: usize = 0; + + // A helper macro to push bits into our accumulator and flush bytes when possible. + macro_rules! push_bits { + ($value:expr, $num_bits:expr) => {{ + // Append the new bits to the accumulator. + acc |= ($value as u64) << bits_in_acc; + bits_in_acc += $num_bits; + // While we have a full byte, flush it. + while bits_in_acc >= 8 { + output.push((acc & 0xFF) as u8); + acc >>= 8; + bits_in_acc -= 8; + } + }}; + } + + for scalar in scalars { + if scalar.len() != 56 { + return vec![]; + } + + for &byte in &scalar[1..55] { + push_bits!(byte, 8); + } + } + + output +} diff --git a/crates/verenc/src/ve.rs b/crates/verenc/src/ve.rs new file mode 100644 index 0000000..0589fff --- /dev/null +++ b/crates/verenc/src/ve.rs @@ -0,0 +1,30 @@ +pub trait VerEnc { + type SystemParams; + type Statement; + type Witness; + type PKE; + type EncKey; + type DecKey; + type VEParams; + type VEProof; + type VECipherText; + + fn setup(params: &Self::SystemParams, vparams: &Self::VEParams, pke: Self::PKE) -> Self; + + fn set_ek(&mut self, ek: Self::EncKey); + + fn kgen(&mut self) -> Self::DecKey; + + fn get_public_key(&self) -> &Self::EncKey; + + fn igen(&self, w: &[u8;56]) -> (Self::Statement, Self::Witness); + + fn prove(&self, stm: &Self::Statement, wit: &Self::Witness) -> Self::VEProof; + + fn verify(&self, stm: &Self::Statement, pi: &Self::VEProof) -> bool; + + fn compress(&self, stm: &Self::Statement, pi: &Self::VEProof) -> Self::VECipherText; + + fn recover(&self, stm: &Self::Statement, dk: &Self::DecKey, ve_ct: &Self::VECipherText) -> Self::Witness; + +} diff --git a/node/app/wire.go b/node/app/wire.go index 990cf86..c7a8d4f 100644 --- a/node/app/wire.go +++ b/node/app/wire.go @@ -58,11 +58,13 @@ var storeSet = wire.NewSet( store.NewPebbleCoinStore, store.NewPebbleKeyStore, store.NewPebbleDataProofStore, + store.NewPebbleHypergraphStore, store.NewPeerstoreDatastore, wire.Bind(new(store.ClockStore), new(*store.PebbleClockStore)), wire.Bind(new(store.CoinStore), new(*store.PebbleCoinStore)), wire.Bind(new(store.KeyStore), new(*store.PebbleKeyStore)), wire.Bind(new(store.DataProofStore), new(*store.PebbleDataProofStore)), + wire.Bind(new(store.HypergraphStore), new(*store.PebbleHypergraphStore)), wire.Bind(new(store.Peerstore), new(*store.PeerstoreDatastore)), ) diff --git a/node/app/wire_gen.go b/node/app/wire_gen.go index a8da265..d357b7f 100644 --- a/node/app/wire_gen.go +++ b/node/app/wire_gen.go @@ -47,11 +47,12 @@ func NewDebugNode(configConfig *config.Config, selfTestReport *protobufs.SelfTes blossomSub := p2p.NewBlossomSub(p2PConfig, zapLogger) frameProver := crypto.NewCachedWesolowskiFrameProver(zapLogger) kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger) + pebbleHypergraphStore := store.NewPebbleHypergraphStore(pebbleDB, zapLogger) engineConfig := configConfig.Engine masterTimeReel := time.NewMasterTimeReel(zapLogger, pebbleClockStore, engineConfig, frameProver) inMemoryPeerInfoManager := p2p.NewInMemoryPeerInfoManager(zapLogger) pebbleKeyStore := store.NewPebbleKeyStore(pebbleDB, zapLogger) - tokenExecutionEngine := token.NewTokenExecutionEngine(zapLogger, configConfig, fileKeyManager, blossomSub, frameProver, kzgInclusionProver, pebbleClockStore, pebbleDataProofStore, pebbleCoinStore, masterTimeReel, inMemoryPeerInfoManager, pebbleKeyStore, selfTestReport) + tokenExecutionEngine := token.NewTokenExecutionEngine(zapLogger, configConfig, fileKeyManager, blossomSub, frameProver, kzgInclusionProver, pebbleClockStore, pebbleDataProofStore, pebbleHypergraphStore, pebbleCoinStore, masterTimeReel, inMemoryPeerInfoManager, pebbleKeyStore, selfTestReport) masterClockConsensusEngine := master.NewMasterClockConsensusEngine(engineConfig, zapLogger, pebbleClockStore, fileKeyManager, blossomSub, kzgInclusionProver, frameProver, masterTimeReel, inMemoryPeerInfoManager, selfTestReport) node, err := newNode(zapLogger, pebbleDataProofStore, pebbleClockStore, pebbleCoinStore, fileKeyManager, blossomSub, tokenExecutionEngine, masterClockConsensusEngine, pebbleDB) if err != nil { @@ -73,11 +74,12 @@ func NewNode(configConfig *config.Config, selfTestReport *protobufs.SelfTestRepo blossomSub := p2p.NewBlossomSub(p2PConfig, zapLogger) frameProver := crypto.NewCachedWesolowskiFrameProver(zapLogger) kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger) + pebbleHypergraphStore := store.NewPebbleHypergraphStore(pebbleDB, zapLogger) engineConfig := configConfig.Engine masterTimeReel := time.NewMasterTimeReel(zapLogger, pebbleClockStore, engineConfig, frameProver) inMemoryPeerInfoManager := p2p.NewInMemoryPeerInfoManager(zapLogger) pebbleKeyStore := store.NewPebbleKeyStore(pebbleDB, zapLogger) - tokenExecutionEngine := token.NewTokenExecutionEngine(zapLogger, configConfig, fileKeyManager, blossomSub, frameProver, kzgInclusionProver, pebbleClockStore, pebbleDataProofStore, pebbleCoinStore, masterTimeReel, inMemoryPeerInfoManager, pebbleKeyStore, selfTestReport) + tokenExecutionEngine := token.NewTokenExecutionEngine(zapLogger, configConfig, fileKeyManager, blossomSub, frameProver, kzgInclusionProver, pebbleClockStore, pebbleDataProofStore, pebbleHypergraphStore, pebbleCoinStore, masterTimeReel, inMemoryPeerInfoManager, pebbleKeyStore, selfTestReport) masterClockConsensusEngine := master.NewMasterClockConsensusEngine(engineConfig, zapLogger, pebbleClockStore, fileKeyManager, blossomSub, kzgInclusionProver, frameProver, masterTimeReel, inMemoryPeerInfoManager, selfTestReport) node, err := newNode(zapLogger, pebbleDataProofStore, pebbleClockStore, pebbleCoinStore, fileKeyManager, blossomSub, tokenExecutionEngine, masterClockConsensusEngine, pebbleDB) if err != nil { @@ -132,7 +134,7 @@ var debugLoggerSet = wire.NewSet( var keyManagerSet = wire.NewSet(wire.FieldsOf(new(*config.Config), "Key"), keys.NewFileKeyManager, wire.Bind(new(keys.KeyManager), new(*keys.FileKeyManager))) -var storeSet = wire.NewSet(wire.FieldsOf(new(*config.Config), "DB"), store.NewPebbleDB, wire.Bind(new(store.KVDB), new(*store.PebbleDB)), store.NewPebbleClockStore, store.NewPebbleCoinStore, store.NewPebbleKeyStore, store.NewPebbleDataProofStore, store.NewPeerstoreDatastore, wire.Bind(new(store.ClockStore), new(*store.PebbleClockStore)), wire.Bind(new(store.CoinStore), new(*store.PebbleCoinStore)), wire.Bind(new(store.KeyStore), new(*store.PebbleKeyStore)), wire.Bind(new(store.DataProofStore), new(*store.PebbleDataProofStore)), wire.Bind(new(store.Peerstore), new(*store.PeerstoreDatastore))) +var storeSet = wire.NewSet(wire.FieldsOf(new(*config.Config), "DB"), store.NewPebbleDB, wire.Bind(new(store.KVDB), new(*store.PebbleDB)), store.NewPebbleClockStore, store.NewPebbleCoinStore, store.NewPebbleKeyStore, store.NewPebbleDataProofStore, store.NewPebbleHypergraphStore, store.NewPeerstoreDatastore, wire.Bind(new(store.ClockStore), new(*store.PebbleClockStore)), wire.Bind(new(store.CoinStore), new(*store.PebbleCoinStore)), wire.Bind(new(store.KeyStore), new(*store.PebbleKeyStore)), wire.Bind(new(store.DataProofStore), new(*store.PebbleDataProofStore)), wire.Bind(new(store.HypergraphStore), new(*store.PebbleHypergraphStore)), wire.Bind(new(store.Peerstore), new(*store.PeerstoreDatastore))) var pubSubSet = wire.NewSet(wire.FieldsOf(new(*config.Config), "P2P"), p2p.NewInMemoryPeerInfoManager, p2p.NewBlossomSub, wire.Bind(new(p2p.PubSub), new(*p2p.BlossomSub)), wire.Bind(new(p2p.PeerInfoManager), new(*p2p.InMemoryPeerInfoManager))) diff --git a/node/build.sh b/node/build.sh index 1a490e9..12433b8 100755 --- a/node/build.sh +++ b/node/build.sh @@ -19,14 +19,14 @@ case "$os_type" in # Check if the architecture is ARM if [[ "$(uname -m)" == "arm64" ]]; then # MacOS ld doesn't support -Bstatic and -Bdynamic, so it's important that there is only a static version of the library - go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -lbls48581 -lvdf -ldl -lm -lflint -lgmp -lmpfr'" "$@" + go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -lbls48581 -lvdf -lverenc -ldl -lm -lflint -lgmp -lmpfr'" "$@" else echo "Unsupported platform" exit 1 fi ;; "Linux") - export CGO_LDFLAGS="-L/usr/local/lib -lflint -lgmp -lmpfr -ldl -lm -L$BINARIES_DIR -lvdf -lbls48581 -static" + export CGO_LDFLAGS="-L/usr/local/lib -lflint -lgmp -lmpfr -ldl -lm -L$BINARIES_DIR -lvdf -lverenc -lbls48581 -static" go build -ldflags "-linkmode 'external'" "$@" ;; *) diff --git a/node/consensus/data/peer_messaging.go b/node/consensus/data/peer_messaging.go index 9c0afd9..89a6b3c 100644 --- a/node/consensus/data/peer_messaging.go +++ b/node/consensus/data/peer_messaging.go @@ -458,13 +458,11 @@ func (e *DataClockConsensusEngine) handleMint( return nil, errors.Wrap(err, "handle mint") } returnAddr = proofAddr - stateTree := &crypto.VectorCommitmentTree{} err = e.coinStore.PutPreCoinProof( txn, head.FrameNumber, proofAddr, add, - stateTree, ) if err != nil { txn.Abort() @@ -503,13 +501,11 @@ func (e *DataClockConsensusEngine) handleMint( return nil, errors.Wrap(err, "handle mint") } returnAddr = proofAddr - stateTree := &crypto.VectorCommitmentTree{} err = e.coinStore.PutPreCoinProof( txn, head.FrameNumber, proofAddr, proof, - stateTree, ) if err != nil { txn.Abort() @@ -555,12 +551,10 @@ func (e *DataClockConsensusEngine) handleMint( txn.Abort() return nil, errors.Wrap(err, "handle mint") } - stateTree := &crypto.VectorCommitmentTree{} e.coinStore.DeletePreCoinProof( txn, a, deletes[0].GetDeletedProof(), - stateTree, ) } if err := txn.Commit(); err != nil { diff --git a/node/consensus/data/token_handle_mint_test.go b/node/consensus/data/token_handle_mint_test.go index 25ae47e..00926f2 100644 --- a/node/consensus/data/token_handle_mint_test.go +++ b/node/consensus/data/token_handle_mint_test.go @@ -665,31 +665,30 @@ func TestHandlePreMidnightMint(t *testing.T) { assert.Len(t, success.Requests, 1) assert.Len(t, fail.Requests, 1) - stateTree := &qcrypto.VectorCommitmentTree{} txn, _ := app.CoinStore.NewTransaction(false) for i, o := range app.TokenOutputs.Outputs { switch e := o.Output.(type) { case *protobufs.TokenOutput_Coin: a, err := GetAddressOfCoin(e.Coin, 1, uint64(i)) assert.NoError(t, err) - err = app.CoinStore.PutCoin(txn, 1, a, e.Coin, stateTree) + err = app.CoinStore.PutCoin(txn, 1, a, e.Coin) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(nil, e.DeletedCoin.Address) assert.NoError(t, err) - err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) assert.NoError(t, err) case *protobufs.TokenOutput_Proof: a, err := GetAddressOfPreCoinProof(e.Proof) assert.NoError(t, err) - err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree) + err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedProof: a, err := GetAddressOfPreCoinProof(e.DeletedProof) assert.NoError(t, err) c, err := app.CoinStore.GetPreCoinProofByAddress(a) assert.NoError(t, err) - err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + err = app.CoinStore.DeletePreCoinProof(txn, a, c) assert.NoError(t, err) } } diff --git a/node/crypto/proof_tree.go b/node/crypto/proof_tree.go index a712e12..6ad3fce 100644 --- a/node/crypto/proof_tree.go +++ b/node/crypto/proof_tree.go @@ -7,6 +7,7 @@ import ( "encoding/gob" "errors" "fmt" + "math/big" rbls48581 "source.quilibrium.com/quilibrium/monorepo/bls48581" ) @@ -24,18 +25,24 @@ const ( type VectorCommitmentNode interface { Commit(recalculate bool) []byte + GetSize() *big.Int } type VectorCommitmentLeafNode struct { Key []byte Value []byte + HashTarget []byte Commitment []byte + Size *big.Int } type VectorCommitmentBranchNode struct { - Prefix []int - Children [BranchNodes]VectorCommitmentNode - Commitment []byte + Prefix []int + Children [BranchNodes]VectorCommitmentNode + Commitment []byte + Size *big.Int + LeafCount int + LongestBranch int } func (n *VectorCommitmentLeafNode) Commit(recalculate bool) []byte { @@ -43,12 +50,20 @@ func (n *VectorCommitmentLeafNode) Commit(recalculate bool) []byte { h := sha512.New() h.Write([]byte{0}) h.Write(n.Key) - h.Write(n.Value) + if len(n.HashTarget) != 0 { + h.Write(n.HashTarget) + } else { + h.Write(n.Value) + } n.Commitment = h.Sum(nil) } return n.Commitment } +func (n *VectorCommitmentLeafNode) GetSize() *big.Int { + return n.Size +} + func (n *VectorCommitmentBranchNode) Commit(recalculate bool) []byte { if n.Commitment == nil || recalculate { data := []byte{} @@ -130,6 +145,10 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool { return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 64) } +func (n *VectorCommitmentBranchNode) GetSize() *big.Int { + return n.Size +} + func (n *VectorCommitmentBranchNode) Prove(index int) []byte { data := []byte{} for _, child := range n.Children { @@ -204,13 +223,44 @@ func getNibblesUntilDiverge(key1, key2 []byte, startDepth int) ([]int, int) { } } -// getLastNibble returns the final nibble after applying a prefix -func getLastNibble(key []byte, prefixLen int) int { - return getNextNibble(key, prefixLen*BranchBits) +func recalcMetadata(node VectorCommitmentNode) ( + leafCount int, + longestBranch int, + size *big.Int, +) { + switch n := node.(type) { + case *VectorCommitmentLeafNode: + // A leaf counts as one, and its depth (from itself) is zero. + return 1, 0, n.Size + case *VectorCommitmentBranchNode: + totalLeaves := 0 + maxChildDepth := 0 + size := new(big.Int) + for _, child := range n.Children { + if child != nil { + cLeaves, cDepth, cSize := recalcMetadata(child) + totalLeaves += cLeaves + size.Add(size, cSize) + if cDepth > maxChildDepth { + maxChildDepth = cDepth + } + } + } + // Store the aggregated values in the branch node. + n.LeafCount = totalLeaves + // The branch’s longest branch is one more than its deepest child. + n.LongestBranch = maxChildDepth + 1 + n.Size = size + return totalLeaves, n.LongestBranch, n.Size + } + return 0, 0, new(big.Int) } // Insert adds or updates a key-value pair in the tree -func (t *VectorCommitmentTree) Insert(key, value []byte) error { +func (t *VectorCommitmentTree) Insert( + key, value, hashTarget []byte, + size *big.Int, +) error { if len(key) == 0 { return errors.New("empty key not allowed") } @@ -218,14 +268,21 @@ func (t *VectorCommitmentTree) Insert(key, value []byte) error { var insert func(node VectorCommitmentNode, depth int) VectorCommitmentNode insert = func(node VectorCommitmentNode, depth int) VectorCommitmentNode { if node == nil { - return &VectorCommitmentLeafNode{Key: key, Value: value} + return &VectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + } } switch n := node.(type) { case *VectorCommitmentLeafNode: if bytes.Equal(n.Key, key) { n.Value = value + n.HashTarget = hashTarget n.Commitment = nil + n.Size = size return n } @@ -241,7 +298,12 @@ func (t *VectorCommitmentTree) Insert(key, value []byte) error { finalOldNibble := getNextNibble(n.Key, divergeDepth) finalNewNibble := getNextNibble(key, divergeDepth) branch.Children[finalOldNibble] = n - branch.Children[finalNewNibble] = &VectorCommitmentLeafNode{Key: key, Value: value} + branch.Children[finalNewNibble] = &VectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + } return branch @@ -258,21 +320,32 @@ func (t *VectorCommitmentTree) Insert(key, value []byte) error { // Position old branch and new leaf newBranch.Children[expectedNibble] = n n.Prefix = n.Prefix[i+1:] // remove shared prefix from old branch - newBranch.Children[actualNibble] = &VectorCommitmentLeafNode{Key: key, Value: value} + newBranch.Children[actualNibble] = &VectorCommitmentLeafNode{ + Key: key, + Value: value, + HashTarget: hashTarget, + Size: size, + } + recalcMetadata(newBranch) return newBranch } } // Key matches prefix, continue with final nibble finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits) - n.Children[finalNibble] = insert(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits) + n.Children[finalNibble] = insert( + n.Children[finalNibble], + depth+len(n.Prefix)*BranchBits+BranchBits, + ) n.Commitment = nil + recalcMetadata(n) return n } else { // Simple branch without prefix nibble := getNextNibble(key, depth) n.Children[nibble] = insert(n.Children[nibble], depth+BranchBits) n.Commitment = nil + recalcMetadata(n) return n } } @@ -456,9 +529,10 @@ func (t *VectorCommitmentTree) Delete(key []byte) error { } } + var retNode VectorCommitmentNode switch childCount { case 0: - return nil + retNode = nil case 1: if childBranch, ok := lastChild.(*VectorCommitmentBranchNode); ok { // Merge: @@ -470,13 +544,19 @@ func (t *VectorCommitmentTree) Delete(key []byte) error { childBranch.Prefix = mergedPrefix childBranch.Commitment = nil - return childBranch + retNode = childBranch + } else { + retNode = lastChild } - - return lastChild default: - return n + retNode = n } + + if branch, ok := retNode.(*VectorCommitmentBranchNode); ok { + recalcMetadata(branch) + } + + return retNode default: return node } @@ -486,6 +566,18 @@ func (t *VectorCommitmentTree) Delete(key []byte) error { return nil } +func (t *VectorCommitmentTree) GetMetadata() (leafCount int, longestBranch int) { + switch root := t.Root.(type) { + case nil: + return 0, 0 + case *VectorCommitmentLeafNode: + return 1, 0 + case *VectorCommitmentBranchNode: + return root.LeafCount, root.LongestBranch + } + return 0, 0 +} + // Commit returns the root of the tree func (t *VectorCommitmentTree) Commit(recalculate bool) []byte { if t.Root == nil { @@ -494,7 +586,11 @@ func (t *VectorCommitmentTree) Commit(recalculate bool) []byte { return t.Root.Commit(recalculate) } -func debugNode(node VectorCommitmentNode, depth int, prefix string) { +func (t *VectorCommitmentTree) GetSize() *big.Int { + return t.Root.GetSize() +} + +func DebugNode(node VectorCommitmentNode, depth int, prefix string) { if node == nil { return } @@ -507,7 +603,7 @@ func debugNode(node VectorCommitmentNode, depth int, prefix string) { for i, child := range n.Children { if child != nil { fmt.Printf("%s [%d]:\n", prefix, i) - debugNode(child, depth+1, prefix+" ") + DebugNode(child, depth+1, prefix+" ") } } } diff --git a/node/crypto/proof_tree_test.go b/node/crypto/proof_tree_test.go index f855398..7f12769 100644 --- a/node/crypto/proof_tree_test.go +++ b/node/crypto/proof_tree_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "fmt" + "math/big" "testing" "source.quilibrium.com/quilibrium/monorepo/bls48581/generated/bls48581" @@ -17,7 +18,7 @@ func BenchmarkVectorCommitmentTreeInsert(b *testing.B) { d := make([]byte, 32) rand.Read(d) addresses = append(addresses, d) - err := tree.Insert(d, d) + err := tree.Insert(d, d, nil, big.NewInt(1)) if err != nil { b.Errorf("Failed to insert item %d: %v", i, err) } @@ -32,7 +33,7 @@ func BenchmarkVectorCommitmentTreeCommit(b *testing.B) { d := make([]byte, 32) rand.Read(d) addresses = append(addresses, d) - err := tree.Insert(d, d) + err := tree.Insert(d, d, nil, big.NewInt(1)) if err != nil { b.Errorf("Failed to insert item %d: %v", i, err) } @@ -48,7 +49,7 @@ func BenchmarkVectorCommitmentTreeProve(b *testing.B) { d := make([]byte, 32) rand.Read(d) addresses = append(addresses, d) - err := tree.Insert(d, d) + err := tree.Insert(d, d, nil, big.NewInt(1)) if err != nil { b.Errorf("Failed to insert item %d: %v", i, err) } @@ -64,7 +65,7 @@ func BenchmarkVectorCommitmentTreeVerify(b *testing.B) { d := make([]byte, 32) rand.Read(d) addresses = append(addresses, d) - err := tree.Insert(d, d) + err := tree.Insert(d, d, nil, big.NewInt(1)) if err != nil { b.Errorf("Failed to insert item %d: %v", i, err) } @@ -80,13 +81,13 @@ func TestVectorCommitmentTrees(t *testing.T) { tree := &VectorCommitmentTree{} // Test single insert - err := tree.Insert([]byte("key1"), []byte("value1")) + err := tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1)) if err != nil { t.Errorf("Failed to insert: %v", err) } // Test duplicate key - err = tree.Insert([]byte("key1"), []byte("value2")) + err = tree.Insert([]byte("key1"), []byte("value2"), nil, big.NewInt(1)) if err != nil { t.Errorf("Failed to update existing key: %v", err) } @@ -100,7 +101,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Test empty key - err = tree.Insert([]byte{}, []byte("value")) + err = tree.Insert([]byte{}, []byte("value"), nil, big.NewInt(1)) if err == nil { t.Error("Expected error for empty key, got none") } @@ -114,7 +115,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Insert and get - tree.Insert([]byte("key1"), []byte("value1")) + tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1)) value, err = tree.Get([]byte("key1")) if err != nil { t.Errorf("Failed to get value: %v", err) @@ -138,7 +139,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Insert and delete - tree.Insert([]byte("key1"), []byte("value1")) + tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1)) err = tree.Delete([]byte("key1")) if err != nil { t.Errorf("Failed to delete: %v", err) @@ -167,7 +168,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } for i, key := range keys { - err := tree.Insert([]byte(key), []byte("value"+string(rune('1'+i)))) + err := tree.Insert([]byte(key), []byte("value"+string(rune('1'+i))), nil, big.NewInt(1)) if err != nil { t.Errorf("Failed to insert key %s: %v", key, err) } @@ -220,7 +221,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Root should change after insert - tree.Insert([]byte("key1"), []byte("value1")) + tree.Insert([]byte("key1"), []byte("value1"), nil, big.NewInt(1)) firstRoot := tree.Root.Commit(false) if bytes.Equal(firstRoot, bytes.Repeat([]byte{0x00}, 64)) { @@ -228,7 +229,7 @@ func TestVectorCommitmentTrees(t *testing.T) { } // Root should change after update - tree.Insert([]byte("key1"), []byte("value2")) + tree.Insert([]byte("key1"), []byte("value2"), nil, big.NewInt(1)) secondRoot := tree.Root.Commit(false) if bytes.Equal(secondRoot, firstRoot) { @@ -271,17 +272,21 @@ func TestVectorCommitmentTrees(t *testing.T) { for i := 0; i < 10000; i++ { key := addresses[i] value := addresses[i] - err := tree.Insert(key, value) + err := tree.Insert(key, value, nil, big.NewInt(1)) if err != nil { t.Errorf("Failed to insert item %d: %v", i, err) } } + if tree.GetSize().Cmp(big.NewInt(10000)) != 0 { + t.Errorf("invalid tree size: %s", tree.GetSize().String()) + } + // Insert 10000 items in reverse for i := 9999; i >= 0; i-- { key := addresses[i] value := addresses[i] - err := cmptree.Insert(key, value) + err := cmptree.Insert(key, value, nil, big.NewInt(1)) if err != nil { t.Errorf("Failed to insert item %d: %v", i, err) } @@ -314,15 +319,23 @@ func TestVectorCommitmentTrees(t *testing.T) { tree.Delete(key) } + if tree.GetSize().Cmp(big.NewInt(5000)) != 0 { + t.Errorf("invalid tree size: %s", tree.GetSize().String()) + } + // add new for i := 0; i < 5000; i++ { - tree.Insert(newAdditions[i], newAdditions[i]) + tree.Insert(newAdditions[i], newAdditions[i], nil, big.NewInt(1)) + } + + if tree.GetSize().Cmp(big.NewInt(10000)) != 0 { + t.Errorf("invalid tree size: %s", tree.GetSize().String()) } cmptree = &VectorCommitmentTree{} for i := 0; i < 10000; i++ { - cmptree.Insert(kept[i], kept[i]) + cmptree.Insert(kept[i], kept[i], nil, big.NewInt(1)) } // Verify all items for i := 0; i < 10000; i++ { @@ -355,7 +368,16 @@ func TestVectorCommitmentTrees(t *testing.T) { t.Errorf("proof failed") } - for _, p := range proofs { - fmt.Printf("%x\n", p) + leaves, longestBranch := tree.GetMetadata() + + if leaves != 10000 { + t.Errorf("incorrect leaf count, %d, %d,", 10000, leaves) } + + // Statistical assumption, can be flaky + if longestBranch != 4 { + t.Errorf("incorrect longest branch count, %d, %d,", 4, longestBranch) + } + + DebugNode(tree.Root, 0, "") } diff --git a/node/crypto/tree_compare.go b/node/crypto/tree_compare.go index 4893647..a52c744 100644 --- a/node/crypto/tree_compare.go +++ b/node/crypto/tree_compare.go @@ -152,8 +152,8 @@ type LeafDifference struct { // CompareLeaves returns all leaves that differ between the two trees func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { // Get all leaves from both trees - leaves1 := getAllLeaves(tree1.Root) - leaves2 := getAllLeaves(tree2.Root) + leaves1 := GetAllLeaves(tree1.Root) + leaves2 := GetAllLeaves(tree2.Root) differences := make([]LeafDifference, 0) @@ -206,8 +206,8 @@ func CompareLeaves(tree1, tree2 *VectorCommitmentTree) []LeafDifference { return differences } -// getAllLeaves returns all leaf nodes in the tree -func getAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { +// GetAllLeaves returns all leaf nodes in the tree +func GetAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { if node == nil { return nil } @@ -220,7 +220,7 @@ func getAllLeaves(node VectorCommitmentNode) []*VectorCommitmentLeafNode { case *VectorCommitmentBranchNode: for _, child := range n.Children { if child != nil { - childLeaves := getAllLeaves(child) + childLeaves := GetAllLeaves(child) leaves = append(leaves, childLeaves...) } } diff --git a/node/crypto/verifiable_encryption.go b/node/crypto/verifiable_encryption.go new file mode 100644 index 0000000..c4dafce --- /dev/null +++ b/node/crypto/verifiable_encryption.go @@ -0,0 +1,231 @@ +package crypto + +import ( + "encoding/binary" + "sync" + + "source.quilibrium.com/quilibrium/monorepo/verenc" + generated "source.quilibrium.com/quilibrium/monorepo/verenc/generated/verenc" +) + +type VerEnc interface { + ToBytes() []byte + GetStatement() []byte + Verify(proof []byte) bool +} + +type VerEncProof interface { + ToBytes() []byte + Compress() VerEnc + Verify() bool +} + +type VerifiableEncryptor interface { + Encrypt( + data []byte, + publicKey []byte, + ) []VerEncProof + Decrypt( + encrypted []VerEnc, + decryptionKey []byte, + ) []byte +} + +var _ VerifiableEncryptor = (*MPCitHVerifiableEncryptor)(nil) + +type MPCitHVerEncProof struct { + generated.VerencProof +} + +type MPCitHVerEnc struct { + generated.CompressedCiphertext + BlindingPubkey []uint8 + Statement []uint8 +} + +func MPCitHVerEncProofFromBytes(data []byte) MPCitHVerEncProof { + if len(data) != 9012 { + return MPCitHVerEncProof{} + } + + polycom := [][]byte{} + for i := 0; i < 23; i++ { + polycom = append(polycom, data[235+(i*57):292+(i*57)]) + } + + ctexts := []generated.VerencCiphertext{} + srs := []generated.VerencShare{} + + for i := 0; i < 42; i++ { + ctexts = append(ctexts, generated.VerencCiphertext{ + C1: data[1546+(i*(57+56+4)) : 1603+(i*(57+56+4))], + C2: data[1603+(i*(57+56+4)) : 1659+(i*(57+56+4))], + I: binary.BigEndian.Uint64(data[1659+(i*(57+56+4)) : 1663+(i*(57+56+4))]), + }) + } + + for i := 0; i < 22; i++ { + srs = append(srs, generated.VerencShare{ + S1: data[6460+(i*(56+56+4)) : 6516+(i*(56+56+4))], + S2: data[6516+(i*(56+56+4)) : 6572+(i*(56+56+4))], + I: binary.BigEndian.Uint64(data[6572+(i*(56+56+4)) : 6576+(i*(56+56+4))]), + }) + } + + return MPCitHVerEncProof{ + generated.VerencProof{ + BlindingPubkey: data[:57], + EncryptionKey: data[57:114], + Statement: data[114:171], + Challenge: data[171:235], + Polycom: polycom, + Ctexts: ctexts, + SharesRands: srs, + }, + } +} + +func (p MPCitHVerEncProof) ToBytes() []byte { + output := []byte{} + output = append(output, p.BlindingPubkey...) + output = append(output, p.EncryptionKey...) + output = append(output, p.Statement...) + output = append(output, p.Challenge...) + + for _, pol := range p.Polycom { + output = append(output, pol...) + } + + for _, ct := range p.Ctexts { + output = append(output, ct.C1...) + output = append(output, ct.C2...) + output = binary.BigEndian.AppendUint64(output, ct.I) + } + + for _, sr := range p.SharesRands { + output = append(output, sr.S1...) + output = append(output, sr.S2...) + output = binary.BigEndian.AppendUint64(output, sr.I) + } + + return output +} + +func (p MPCitHVerEncProof) Compress() VerEnc { + compressed := verenc.VerencCompress(p.VerencProof) + return MPCitHVerEnc{ + CompressedCiphertext: compressed, + BlindingPubkey: p.BlindingPubkey, + Statement: p.Statement, + } +} + +func (p MPCitHVerEncProof) Verify() bool { + return verenc.VerencVerify(p.VerencProof) +} + +func MPCitHVerEncFromBytes(data []byte) MPCitHVerEnc { + ciphertext := generated.CompressedCiphertext{} + for i := 0; i < 3; i++ { + ciphertext.Ctexts = append(ciphertext.Ctexts, generated.VerencCiphertext{ + C1: data[0+(i*(57+56)) : 57+(i*(57+56))], + C2: data[57+(i*(57+56)) : 113+(i*(57+56))], + }) + ciphertext.Aux = append(ciphertext.Aux, data[507+(i*56):563+(i*56)]) + } + return MPCitHVerEnc{ + CompressedCiphertext: ciphertext, + BlindingPubkey: data[731:788], + Statement: data[788:845], + } +} + +func (e MPCitHVerEnc) ToBytes() []byte { + output := []byte{} + for _, ct := range e.Ctexts { + output = append(output, ct.C1...) + output = append(output, ct.C2...) + } + for _, a := range e.Aux { + output = append(output, a...) + } + output = append(output, e.BlindingPubkey...) + output = append(output, e.Statement...) + return output +} + +func (e MPCitHVerEnc) GetStatement() []byte { + return e.Statement +} + +func (e MPCitHVerEnc) Verify(proof []byte) bool { + proofData := MPCitHVerEncProofFromBytes(proof) + return proofData.Verify() +} + +type MPCitHVerifiableEncryptor struct { + parallelism int +} + +func NewMPCitHVerifiableEncryptor(parallelism int) *MPCitHVerifiableEncryptor { + return &MPCitHVerifiableEncryptor{ + parallelism: parallelism, + } +} + +func (v *MPCitHVerifiableEncryptor) Encrypt( + data []byte, + publicKey []byte, +) []VerEncProof { + chunks := verenc.ChunkDataForVerenc(data) + results := make([]VerEncProof, len(chunks)) + var wg sync.WaitGroup + throttle := make(chan struct{}, v.parallelism) + for i, chunk := range chunks { + throttle <- struct{}{} + wg.Add(1) + go func(chunk []byte, i int) { + defer func() { <-throttle }() + defer wg.Done() + proof := verenc.NewVerencProofEncryptOnly(chunk, publicKey) + results[i] = MPCitHVerEncProof{ + generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }, + } + }(chunk, i) + } + wg.Wait() + return results +} + +func (v *MPCitHVerifiableEncryptor) Decrypt( + encrypted []VerEnc, + decyptionKey []byte, +) []byte { + results := make([][]byte, len(encrypted)) + var wg sync.WaitGroup + throttle := make(chan struct{}, v.parallelism) + for i, chunk := range encrypted { + throttle <- struct{}{} + wg.Add(1) + go func(chunk VerEnc, i int) { + defer func() { <-throttle }() + defer wg.Done() + results[i] = verenc.VerencRecover(generated.VerencDecrypt{ + BlindingPubkey: chunk.(MPCitHVerEnc).BlindingPubkey, + DecryptionKey: decyptionKey, + Statement: chunk.(MPCitHVerEnc).Statement, + Ciphertexts: chunk.(MPCitHVerEnc).CompressedCiphertext, + }) + }(chunk, i) + } + wg.Wait() + return verenc.CombineChunkedData(results) +} diff --git a/node/execution/intrinsics/token/application/token_handle_prover_join_test.go b/node/execution/intrinsics/token/application/token_handle_prover_join_test.go index 5f495e1..85d6134 100644 --- a/node/execution/intrinsics/token/application/token_handle_prover_join_test.go +++ b/node/execution/intrinsics/token/application/token_handle_prover_join_test.go @@ -330,24 +330,23 @@ func TestHandleProverJoin(t *testing.T) { assert.Len(t, success.Requests, 1) assert.Len(t, app.TokenOutputs.Outputs, 1) txn, _ = app.CoinStore.NewTransaction(false) - stateTree := &qcrypto.VectorCommitmentTree{} for i, o := range app.TokenOutputs.Outputs { switch e := o.Output.(type) { case *protobufs.TokenOutput_Coin: a, err := token.GetAddressOfCoin(e.Coin, 1, uint64(i)) assert.NoError(t, err) - err = app.CoinStore.PutCoin(txn, 1, a, e.Coin, stateTree) + err = app.CoinStore.PutCoin(txn, 1, a, e.Coin) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(nil, e.DeletedCoin.Address) assert.NoError(t, err) - err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) assert.NoError(t, err) case *protobufs.TokenOutput_Proof: a, err := token.GetAddressOfPreCoinProof(e.Proof) fmt.Printf("add addr %x\n", a) assert.NoError(t, err) - err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree) + err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedProof: a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) @@ -355,7 +354,7 @@ func TestHandleProverJoin(t *testing.T) { assert.NoError(t, err) c, err := app.CoinStore.GetPreCoinProofByAddress(a) assert.NoError(t, err) - err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + err = app.CoinStore.DeletePreCoinProof(txn, a, c) assert.NoError(t, err) } } @@ -385,18 +384,18 @@ func TestHandleProverJoin(t *testing.T) { case *protobufs.TokenOutput_Coin: a, err := token.GetAddressOfCoin(e.Coin, 4, uint64(i)) assert.NoError(t, err) - err = app.CoinStore.PutCoin(txn, 4, a, e.Coin, stateTree) + err = app.CoinStore.PutCoin(txn, 4, a, e.Coin) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) assert.NoError(t, err) - err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) assert.NoError(t, err) case *protobufs.TokenOutput_Proof: a, err := token.GetAddressOfPreCoinProof(e.Proof) fmt.Printf("add addr %x\n", a) assert.NoError(t, err) - err = app.CoinStore.PutPreCoinProof(txn, 4, a, e.Proof, stateTree) + err = app.CoinStore.PutPreCoinProof(txn, 4, a, e.Proof) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedProof: a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) @@ -404,7 +403,7 @@ func TestHandleProverJoin(t *testing.T) { assert.NoError(t, err) c, err := app.CoinStore.GetPreCoinProofByAddress(a) assert.NoError(t, err) - err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + err = app.CoinStore.DeletePreCoinProof(txn, a, c) assert.NoError(t, err) case *protobufs.TokenOutput_Penalty: // gotPenalty = true @@ -438,19 +437,19 @@ func TestHandleProverJoin(t *testing.T) { case *protobufs.TokenOutput_Coin: a, err := token.GetAddressOfCoin(e.Coin, 5, uint64(i)) assert.NoError(t, err) - err = app.CoinStore.PutCoin(txn, 5, a, e.Coin, stateTree) + err = app.CoinStore.PutCoin(txn, 5, a, e.Coin) assert.NoError(t, err) coins = append(coins, a) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) assert.NoError(t, err) - err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) assert.NoError(t, err) case *protobufs.TokenOutput_Proof: a, err := token.GetAddressOfPreCoinProof(e.Proof) fmt.Printf("add addr %x\n", a) assert.NoError(t, err) - err = app.CoinStore.PutPreCoinProof(txn, 5, a, e.Proof, stateTree) + err = app.CoinStore.PutPreCoinProof(txn, 5, a, e.Proof) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedProof: a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) @@ -458,7 +457,7 @@ func TestHandleProverJoin(t *testing.T) { assert.NoError(t, err) c, err := app.CoinStore.GetPreCoinProofByAddress(a) assert.NoError(t, err) - err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + err = app.CoinStore.DeletePreCoinProof(txn, a, c) assert.NoError(t, err) case *protobufs.TokenOutput_Penalty: // gotPenalty = true @@ -493,25 +492,25 @@ func TestHandleProverJoin(t *testing.T) { case *protobufs.TokenOutput_Coin: a, err := token.GetAddressOfCoin(e.Coin, 5, uint64(i)) assert.NoError(t, err) - err = app.CoinStore.PutCoin(txn, 5, a, e.Coin, stateTree) + err = app.CoinStore.PutCoin(txn, 5, a, e.Coin) assert.NoError(t, err) coins = append(coins, a) case *protobufs.TokenOutput_DeletedCoin: c, err := app.CoinStore.GetCoinByAddress(txn, e.DeletedCoin.Address) assert.NoError(t, err) - err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c, stateTree) + err = app.CoinStore.DeleteCoin(txn, e.DeletedCoin.Address, c) assert.NoError(t, err) case *protobufs.TokenOutput_Proof: a, err := token.GetAddressOfPreCoinProof(e.Proof) assert.NoError(t, err) - err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof, stateTree) + err = app.CoinStore.PutPreCoinProof(txn, 1, a, e.Proof) assert.NoError(t, err) case *protobufs.TokenOutput_DeletedProof: a, err := token.GetAddressOfPreCoinProof(e.DeletedProof) assert.NoError(t, err) c, err := app.CoinStore.GetPreCoinProofByAddress(a) assert.NoError(t, err) - err = app.CoinStore.DeletePreCoinProof(txn, a, c, stateTree) + err = app.CoinStore.DeletePreCoinProof(txn, a, c) assert.NoError(t, err) } } diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 289af7e..c68f8c4 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -4,9 +4,11 @@ import ( "bytes" "context" "crypto" + "encoding/binary" "encoding/hex" "fmt" "math/big" + "runtime" "slices" "strconv" "strings" @@ -27,6 +29,7 @@ import ( qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/execution" "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" + hypergraph "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" "source.quilibrium.com/quilibrium/monorepo/node/internal/frametime" qruntime "source.quilibrium.com/quilibrium/monorepo/node/internal/runtime" "source.quilibrium.com/quilibrium/monorepo/node/keys" @@ -88,6 +91,7 @@ type TokenExecutionEngine struct { logger *zap.Logger clock *data.DataClockConsensusEngine clockStore store.ClockStore + hypergraphStore store.HypergraphStore coinStore store.CoinStore keyStore store.KeyStore keyManager keys.KeyManager @@ -105,7 +109,8 @@ type TokenExecutionEngine struct { intrinsicFilter []byte frameProver qcrypto.FrameProver peerSeniority *PeerSeniority - stateTree *qcrypto.VectorCommitmentTree + hypergraph *hypergraph.Hypergraph + mpcithVerEnc *qcrypto.MPCitHVerifiableEncryptor } func NewTokenExecutionEngine( @@ -117,6 +122,7 @@ func NewTokenExecutionEngine( inclusionProver qcrypto.InclusionProver, clockStore store.ClockStore, dataProofStore store.DataProofStore, + hypergraphStore store.HypergraphStore, coinStore store.CoinStore, masterTimeReel *time.MasterTimeReel, peerInfoManager p2p.PeerInfoManager, @@ -139,7 +145,10 @@ func NewTokenExecutionEngine( var inclusionProof *qcrypto.InclusionAggregateProof var proverKeys [][]byte var peerSeniority map[string]uint64 - stateTree := &qcrypto.VectorCommitmentTree{} + hypergraph := hypergraph.NewHypergraph() + mpcithVerEnc := qcrypto.NewMPCitHVerifiableEncryptor( + runtime.NumCPU(), + ) if err != nil && errors.Is(err, store.ErrNotFound) { origin, inclusionProof, proverKeys, peerSeniority = CreateGenesisState( @@ -149,7 +158,9 @@ func NewTokenExecutionEngine( inclusionProver, clockStore, coinStore, - stateTree, + hypergraphStore, + hypergraph, + mpcithVerEnc, uint(cfg.P2P.Network), ) if err := coinStore.SetMigrationVersion( @@ -176,7 +187,9 @@ func NewTokenExecutionEngine( inclusionProver, clockStore, coinStore, - stateTree, + hypergraphStore, + hypergraph, + mpcithVerEnc, uint(cfg.P2P.Network), ) } @@ -232,6 +245,7 @@ func NewTokenExecutionEngine( alreadyPublishedShare: false, intrinsicFilter: intrinsicFilter, peerSeniority: NewFromMap(peerSeniority), + mpcithVerEnc: mpcithVerEnc, } alwaysSend := false @@ -349,18 +363,18 @@ func NewTokenExecutionEngine( frame, _, err := e.clockStore.GetLatestDataClockFrame(e.intrinsicFilter) if err != nil || frame.FrameNumber < 186405 { - e.rebuildStateTree() + e.rebuildHypergraph() } else { - e.stateTree, err = e.clockStore.GetDataStateTree(e.intrinsicFilter) + e.hypergraph, err = e.hypergraphStore.LoadHypergraph() if err != nil && !errors.Is(err, store.ErrNotFound) { e.logger.Error( - "error encountered while fetching state tree, rebuilding", + "error encountered while fetching hypergraph, rebuilding", zap.Error(err), ) } - if e.stateTree == nil { - e.rebuildStateTree() + if e.hypergraph == nil { + e.rebuildHypergraph() } } @@ -428,19 +442,30 @@ func NewTokenExecutionEngine( var _ execution.ExecutionEngine = (*TokenExecutionEngine)(nil) -func (e *TokenExecutionEngine) rebuildStateTree() { - e.logger.Info("rebuilding state tree") - e.stateTree = &qcrypto.VectorCommitmentTree{} +func (e *TokenExecutionEngine) rebuildHypergraph() { + e.logger.Info("rebuilding hypergraph") + e.hypergraph = hypergraph.NewHypergraph() iter, err := e.coinStore.RangeCoins() if err != nil { panic(err) } for iter.First(); iter.Valid(); iter.Next() { key := make([]byte, len(iter.Key()[2:])) - value := make([]byte, len(iter.Value())) copy(key, iter.Key()[2:]) - copy(value, iter.Value()) - e.stateTree.Insert(key, value) + data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range data { + compressed = append(compressed, d.Compress()) + } + if err := e.hypergraph.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(key), + compressed, + ), + ); err != nil { + panic(err) + } } iter.Close() @@ -450,10 +475,21 @@ func (e *TokenExecutionEngine) rebuildStateTree() { } for iter.First(); iter.Valid(); iter.Next() { key := make([]byte, len(iter.Key()[2:])) - value := make([]byte, len(iter.Value())) copy(key, iter.Key()[2:]) - copy(value, iter.Value()) - e.stateTree.Insert(key, value) + data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range data { + compressed = append(compressed, d.Compress()) + } + if err := e.hypergraph.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(key), + compressed, + ), + ); err != nil { + panic(err) + } } iter.Close() e.logger.Info("saving rebuilt state tree") @@ -465,14 +501,14 @@ func (e *TokenExecutionEngine) rebuildStateTree() { e.logger.Info("committing state tree") - root := e.stateTree.Commit(true) + roots := e.hypergraph.Commit() e.logger.Info( - "committed state tree", - zap.String("root", fmt.Sprintf("%x", root)), + "committed hypergraph state", + zap.String("root", fmt.Sprintf("%x", roots[0])), ) - err = e.clockStore.SetDataStateTree(txn, e.intrinsicFilter, e.stateTree) + err = e.hypergraphStore.SaveHypergraph(txn, e.hypergraph) if err != nil { txn.Abort() panic(err) @@ -649,10 +685,10 @@ func (e *TokenExecutionEngine) ProcessFrame( } wg.Wait() - stateTree, err := e.clockStore.GetDataStateTree(e.intrinsicFilter) + hg, err := e.hypergraphStore.LoadHypergraph() if err != nil { txn.Abort() - return nil, errors.Wrap(err, "process frame") + panic(err) } for i, output := range app.TokenOutputs.Outputs { @@ -668,12 +704,34 @@ func (e *TokenExecutionEngine) ProcessFrame( frame.FrameNumber, address, o.Coin, - stateTree, ) if err != nil { txn.Abort() return nil, errors.Wrap(err, "process frame") } + coinBytes, err := proto.Marshal(o.Coin) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, coinBytes...) + proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(address), + compressed, + ), + ); err != nil { + txn.Abort() + panic(err) + } case *protobufs.TokenOutput_DeletedCoin: coin, err := e.coinStore.GetCoinByAddress(nil, o.DeletedCoin.Address) if err != nil { @@ -684,12 +742,34 @@ func (e *TokenExecutionEngine) ProcessFrame( txn, o.DeletedCoin.Address, coin, - stateTree, ) if err != nil { txn.Abort() return nil, errors.Wrap(err, "process frame") } + coinBytes, err := proto.Marshal(coin) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, coinBytes...) + proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.RemoveVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(o.DeletedCoin.Address), + compressed, + ), + ); err != nil { + txn.Abort() + panic(err) + } case *protobufs.TokenOutput_Proof: address, err := outputAddresses[i], outputAddressErrors[i] if err != nil { @@ -701,12 +781,34 @@ func (e *TokenExecutionEngine) ProcessFrame( frame.FrameNumber, address, o.Proof, - stateTree, ) if err != nil { txn.Abort() return nil, errors.Wrap(err, "process frame") } + proofBytes, err := proto.Marshal(o.Proof) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, proofBytes...) + proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(address), + compressed, + ), + ); err != nil { + txn.Abort() + panic(err) + } if len(o.Proof.Amount) == 32 && !bytes.Equal(o.Proof.Amount, make([]byte, 32)) && o.Proof.Commitment != nil { @@ -739,12 +841,34 @@ func (e *TokenExecutionEngine) ProcessFrame( txn, address, o.DeletedProof, - stateTree, ) if err != nil { txn.Abort() return nil, errors.Wrap(err, "process frame") } + proofBytes, err := proto.Marshal(o.DeletedProof) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, proofBytes...) + proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.RemoveVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(address), + compressed, + ), + ); err != nil { + txn.Abort() + panic(err) + } case *protobufs.TokenOutput_Announce: peerIds := []string{} for _, sig := range o.Announce.PublicKeySignaturesEd448 { @@ -1057,24 +1181,23 @@ func (e *TokenExecutionEngine) ProcessFrame( e.logger.Info("committing state tree") - root := stateTree.Commit(false) + roots := hg.Commit() e.logger.Info( - "commited state tree", - zap.String("root", fmt.Sprintf("%x", root)), + "commited hypergraph", + zap.String("root", fmt.Sprintf("%x", roots[0])), ) - err = e.clockStore.SetDataStateTree( + err = e.hypergraphStore.SaveHypergraph( txn, - e.intrinsicFilter, - stateTree, + hg, ) if err != nil { txn.Abort() return nil, errors.Wrap(err, "process frame") } - e.stateTree = stateTree + e.hypergraph = hg return app.Tries, nil } diff --git a/node/execution/intrinsics/token/token_genesis.go b/node/execution/intrinsics/token/token_genesis.go index 1c29336..e16aa4e 100644 --- a/node/execution/intrinsics/token/token_genesis.go +++ b/node/execution/intrinsics/token/token_genesis.go @@ -2,6 +2,7 @@ package token import ( _ "embed" + "encoding/binary" "encoding/hex" "encoding/json" "fmt" @@ -17,9 +18,9 @@ import ( "google.golang.org/protobuf/proto" "source.quilibrium.com/quilibrium/monorepo/nekryptology/pkg/vdf" "source.quilibrium.com/quilibrium/monorepo/node/config" - "source.quilibrium.com/quilibrium/monorepo/node/crypto" qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" + hypergraph "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" "source.quilibrium.com/quilibrium/monorepo/node/p2p" "source.quilibrium.com/quilibrium/monorepo/node/protobufs" "source.quilibrium.com/quilibrium/monorepo/node/store" @@ -504,7 +505,9 @@ func CreateGenesisState( inclusionProver qcrypto.InclusionProver, clockStore store.ClockStore, coinStore store.CoinStore, - stateTree *crypto.VectorCommitmentTree, + hypergraphStore store.HypergraphStore, + hg *hypergraph.Hypergraph, + mpcithVerEnc *qcrypto.MPCitHVerifiableEncryptor, network uint, ) ( []byte, @@ -865,11 +868,32 @@ func CreateGenesisState( 0, address, output.GetCoin(), - stateTree, ) if err != nil { panic(err) } + coinBytes, err := proto.Marshal(output.GetCoin()) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, coinBytes...) + proofs := mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(address), + compressed, + ), + ); err != nil { + panic(err) + } } if err := txn.Commit(); err != nil { panic(err) @@ -890,7 +914,7 @@ func CreateGenesisState( } intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - err = clockStore.SetDataStateTree(txn, intrinsicFilter, stateTree) + err = hypergraphStore.SaveHypergraph(txn, hg) if err != nil { txn.Abort() panic(err) @@ -1012,14 +1036,35 @@ func CreateGenesisState( 0, address, output.GetCoin(), - stateTree, ) if err != nil { panic(err) } + coinBytes, err := proto.Marshal(output.GetCoin()) + if err != nil { + panic(err) + } + + data := []byte{} + data = binary.BigEndian.AppendUint64(data, 0) + data = append(data, coinBytes...) + proofs := mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + compressed := []hypergraph.Encrypted{} + for _, d := range proofs { + compressed = append(compressed, d.Compress()) + } + if err := hg.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(address), + compressed, + ), + ); err != nil { + panic(err) + } } intrinsicFilter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - err = clockStore.SetDataStateTree(txn, intrinsicFilter, stateTree) + err = hypergraphStore.SaveHypergraph(txn, hg) if err != nil { txn.Abort() panic(err) diff --git a/node/go.mod b/node/go.mod index ad490d3..5c5e122 100644 --- a/node/go.mod +++ b/node/go.mod @@ -11,6 +11,8 @@ replace source.quilibrium.com/quilibrium/monorepo/bls48581 => ../bls48581 replace source.quilibrium.com/quilibrium/monorepo/vdf => ../vdf +replace source.quilibrium.com/quilibrium/monorepo/verenc => ../verenc + replace github.com/multiformats/go-multiaddr => ../go-multiaddr replace github.com/multiformats/go-multiaddr-dns => ../go-multiaddr-dns @@ -38,11 +40,13 @@ require ( source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub v0.0.0-00010101000000-000000000000 source.quilibrium.com/quilibrium/monorepo/nekryptology v0.0.0-00010101000000-000000000000 source.quilibrium.com/quilibrium/monorepo/vdf v0.0.0-00010101000000-000000000000 + source.quilibrium.com/quilibrium/monorepo/verenc v0.0.0-00010101000000-000000000000 ) require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/deiu/gon3 v0.0.0-20230411081920-f0f8f879f597 // indirect + github.com/google/subcommands v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 // indirect diff --git a/node/go.sum b/node/go.sum index 88c4a83..acefbe5 100644 --- a/node/go.sum +++ b/node/go.sum @@ -183,6 +183,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/node/hypergraph/application/hypergraph.go b/node/hypergraph/application/hypergraph.go index 5400ef1..de5aff3 100644 --- a/node/hypergraph/application/hypergraph.go +++ b/node/hypergraph/application/hypergraph.go @@ -1,12 +1,26 @@ package application import ( - "errors" + "bytes" + "crypto/sha512" + "encoding/gob" + "math/big" + "github.com/pkg/errors" + "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/p2p" ) type AtomType string +type PhaseType string + +const ( + VertexAtomType AtomType = "vertex" + HyperedgeAtomType AtomType = "hyperedge" + AddsPhaseType PhaseType = "adds" + RemovesPhaseType PhaseType = "removes" +) + type Location [64]byte // 32 bytes for AppAddress + 32 bytes for DataAddress var ErrInvalidAtomType = errors.New("invalid atom type for set") @@ -17,84 +31,272 @@ var ErrIsExtrinsic = errors.New("is extrinsic") // Extract only needed methods of VEnc interface type Encrypted interface { ToBytes() []byte + GetStatement() []byte Verify(proof []byte) bool } -type Vertex struct { - AppAddress [32]byte - DataAddress [32]byte - Data Encrypted +type Vertex interface { + GetID() [64]byte + GetAtomType() AtomType + GetLocation() Location + GetAppAddress() [32]byte + GetDataAddress() [32]byte + ToBytes() []byte + GetData() []Encrypted + GetSize() *big.Int + Commit() []byte } -type Hyperedge struct { - AppAddress [32]byte - DataAddress [32]byte - Extrinsics map[[64]byte]Atom +type Hyperedge interface { + GetID() [64]byte + GetAtomType() AtomType + GetLocation() Location + GetAppAddress() [32]byte + GetDataAddress() [32]byte + ToBytes() []byte + AddExtrinsic(a Atom) + RemoveExtrinsic(a Atom) + GetExtrinsics() map[[64]byte]Atom + GetSize() *big.Int + Commit() []byte } +type vertex struct { + appAddress [32]byte + dataAddress [32]byte + data []Encrypted + dataTree *crypto.VectorCommitmentTree +} + +type hyperedge struct { + appAddress [32]byte + dataAddress [32]byte + extrinsics map[[64]byte]Atom + extTree *crypto.VectorCommitmentTree +} + +var _ Vertex = (*vertex)(nil) +var _ Hyperedge = (*hyperedge)(nil) + type Atom interface { GetID() [64]byte GetAtomType() AtomType GetLocation() Location GetAppAddress() [32]byte GetDataAddress() [32]byte + GetSize() *big.Int + ToBytes() []byte + Commit() []byte } -func (v *Vertex) GetID() [64]byte { +func atomFromBytes(data []byte) Atom { + tree := &crypto.VectorCommitmentTree{} + var b bytes.Buffer + b.Write(data[65:]) + dec := gob.NewDecoder(&b) + if err := dec.Decode(tree); err != nil { + return nil + } + + if data[0] == 0x00 { + encData := []Encrypted{} + for _, d := range crypto.GetAllLeaves(tree) { + verencData := crypto.MPCitHVerEncFromBytes(d.Value) + encData = append(encData, verencData) + } + return &vertex{ + appAddress: [32]byte(data[1:33]), + dataAddress: [32]byte(data[33:65]), + data: encData, + dataTree: tree, + } + } else { + extrinsics := make(map[[64]byte]Atom) + for _, a := range crypto.GetAllLeaves(tree) { + atom := atomFromBytes(a.Value) + extrinsics[[64]byte(a.Key)] = atom + } + return &hyperedge{ + appAddress: [32]byte(data[1:33]), + dataAddress: [32]byte(data[33:65]), + extrinsics: extrinsics, + extTree: tree, + } + } +} + +func NewVertex( + appAddress [32]byte, + dataAddress [32]byte, + data []Encrypted, +) Vertex { + dataTree := &crypto.VectorCommitmentTree{} + for _, d := range data { + dataBytes := d.ToBytes() + id := sha512.Sum512(dataBytes) + dataTree.Insert(id[:], dataBytes, d.GetStatement(), big.NewInt(int64(len(data)*54))) + } + return &vertex{ + appAddress, + dataAddress, + data, + dataTree, + } +} + +func NewHyperedge( + appAddress [32]byte, + dataAddress [32]byte, +) Hyperedge { + return &hyperedge{ + appAddress: appAddress, + dataAddress: dataAddress, + extrinsics: make(map[[64]byte]Atom), + extTree: &crypto.VectorCommitmentTree{}, + } +} + +func (v *vertex) GetID() [64]byte { id := [64]byte{} - copy(id[:32], v.AppAddress[:]) - copy(id[32:64], v.DataAddress[:]) + copy(id[:32], v.appAddress[:]) + copy(id[32:64], v.dataAddress[:]) return id } -func (v *Vertex) GetAtomType() AtomType { - return "vertex" +func (v *vertex) GetSize() *big.Int { + return big.NewInt(int64(len(v.data) * 54)) } -func (v *Vertex) GetLocation() Location { +func (v *vertex) GetAtomType() AtomType { + return VertexAtomType +} + +func (v *vertex) GetLocation() Location { var loc Location - copy(loc[:32], v.AppAddress[:]) - copy(loc[32:], v.DataAddress[:]) + copy(loc[:32], v.appAddress[:]) + copy(loc[32:], v.dataAddress[:]) return loc } -func (v *Vertex) GetAppAddress() [32]byte { - return v.AppAddress +func (v *vertex) GetAppAddress() [32]byte { + return v.appAddress } -func (v *Vertex) GetDataAddress() [32]byte { - return v.DataAddress +func (v *vertex) GetDataAddress() [32]byte { + return v.dataAddress } -func (h *Hyperedge) GetID() [64]byte { +func (v *vertex) GetData() []Encrypted { + return v.data +} + +func (v *vertex) ToBytes() []byte { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(v.dataTree); err != nil { + return nil + } + return append( + append( + append( + []byte{0x00}, + v.appAddress[:]..., + ), + v.dataAddress[:]..., + ), + buf.Bytes()..., + ) +} + +func (v *vertex) Commit() []byte { + return v.dataTree.Commit(false) +} + +func (h *hyperedge) GetID() [64]byte { id := [64]byte{} - copy(id[:32], h.AppAddress[:]) - copy(id[32:], h.DataAddress[:]) + copy(id[:32], h.appAddress[:]) + copy(id[32:], h.dataAddress[:]) return id } -func (h *Hyperedge) GetAtomType() AtomType { - return "hyperedge" +func (h *hyperedge) GetSize() *big.Int { + return big.NewInt(int64(len(h.extrinsics))) } -func (h *Hyperedge) GetLocation() Location { +func (h *hyperedge) GetAtomType() AtomType { + return HyperedgeAtomType +} + +func (h *hyperedge) GetLocation() Location { var loc Location - copy(loc[:32], h.AppAddress[:]) - copy(loc[32:], h.DataAddress[:]) + copy(loc[:32], h.appAddress[:]) + copy(loc[32:], h.dataAddress[:]) return loc } -func (h *Hyperedge) GetAppAddress() [32]byte { - return h.AppAddress +func (h *hyperedge) GetAppAddress() [32]byte { + return h.appAddress } -func (h *Hyperedge) GetDataAddress() [32]byte { - return h.DataAddress +func (h *hyperedge) GetDataAddress() [32]byte { + return h.dataAddress +} + +func (h *hyperedge) ToBytes() []byte { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(h.extrinsics); err != nil { + return nil + } + return append( + append( + append( + []byte{0x01}, + h.appAddress[:]..., + ), + h.dataAddress[:]..., + ), + buf.Bytes()..., + ) +} + +func (h *hyperedge) AddExtrinsic(a Atom) { + id := a.GetID() + atomType := []byte{0x00} + if a.GetAtomType() == HyperedgeAtomType { + atomType = []byte{0x01} + } + h.extTree.Insert(id[:], append(atomType, id[:]...), nil, a.GetSize()) + h.extrinsics[id] = a +} + +func (h *hyperedge) RemoveExtrinsic(a Atom) { + id := a.GetID() + h.extTree.Delete(id[:]) + delete(h.extrinsics, id) +} + +func (h *hyperedge) GetExtrinsics() map[[64]byte]Atom { + ext := make(map[[64]byte]Atom) + for id := range h.extrinsics { + ext[id] = h.extrinsics[id] + } + return ext +} + +func (h *hyperedge) Commit() []byte { + return h.extTree.Commit(false) } type ShardAddress struct { L1 [3]byte - L2 [64]byte + L2 [32]byte + L3 [32]byte +} + +type ShardKey struct { + L1 [3]byte + L2 [32]byte } func GetShardAddress(a Atom) ShardAddress { @@ -103,59 +305,194 @@ func GetShardAddress(a Atom) ShardAddress { return ShardAddress{ L1: [3]byte(p2p.GetBloomFilterIndices(appAddress[:], 256, 3)), - L2: [64]byte(append(append([]byte{}, appAddress[:]...), dataAddress[:]...)), + L2: [32]byte(append([]byte{}, appAddress[:]...)), + L3: [32]byte(append([]byte{}, dataAddress[:]...)), } } +func GetShardKey(a Atom) ShardKey { + s := GetShardAddress(a) + return ShardKey{L1: s.L1, L2: s.L2} +} + type IdSet struct { + dirty bool atomType AtomType atoms map[[64]byte]Atom + tree *crypto.VectorCommitmentTree } func NewIdSet(atomType AtomType) *IdSet { - return &IdSet{atomType: atomType, atoms: make(map[[64]byte]Atom)} + return &IdSet{ + dirty: false, + atomType: atomType, + atoms: make(map[[64]byte]Atom), + tree: &crypto.VectorCommitmentTree{}, + } +} + +func (set *IdSet) FromBytes(treeData []byte) error { + set.tree = &crypto.VectorCommitmentTree{} + var b bytes.Buffer + b.Write(treeData) + dec := gob.NewDecoder(&b) + if err := dec.Decode(set.tree); err != nil { + return errors.Wrap(err, "load set") + } + + for _, leaf := range crypto.GetAllLeaves(set.tree.Root) { + set.atoms[[64]byte(leaf.Key)] = atomFromBytes(leaf.Value) + } + + return nil +} + +func (set *IdSet) IsDirty() bool { + return set.dirty +} + +func (set *IdSet) ToBytes() []byte { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(set.tree); err != nil { + return nil + } + + return buf.Bytes() } func (set *IdSet) Add(atom Atom) error { if atom.GetAtomType() != set.atomType { return ErrInvalidAtomType } - if _, exists := set.atoms[atom.GetID()]; !exists { - set.atoms[atom.GetID()] = atom - } - return nil + + id := atom.GetID() + set.atoms[id] = atom + set.dirty = true + return set.tree.Insert(id[:], atom.ToBytes(), atom.Commit(), atom.GetSize()) +} + +func (set *IdSet) GetSize() *big.Int { + return set.tree.GetSize() } func (set *IdSet) Delete(atom Atom) bool { - if _, exists := set.atoms[atom.GetID()]; exists { - delete(set.atoms, atom.GetID()) - return true + if atom.GetAtomType() != set.atomType { + return false } - return false + + id := atom.GetID() + if err := set.tree.Delete(id[:]); err != nil { + return false + } + + set.dirty = true + delete(set.atoms, id) + + return true } -func (set *IdSet) Has(atom Atom) bool { - _, exists := set.atoms[atom.GetID()] - return exists +func (set *IdSet) Has(key [64]byte) bool { + _, ok := set.atoms[key] + return ok } type Hypergraph struct { - vertexAdds map[ShardAddress]*IdSet - vertexRemoves map[ShardAddress]*IdSet - hyperedgeAdds map[ShardAddress]*IdSet - hyperedgeRemoves map[ShardAddress]*IdSet + size *big.Int + vertexAdds map[ShardKey]*IdSet + vertexRemoves map[ShardKey]*IdSet + hyperedgeAdds map[ShardKey]*IdSet + hyperedgeRemoves map[ShardKey]*IdSet } func NewHypergraph() *Hypergraph { return &Hypergraph{ - vertexAdds: make(map[ShardAddress]*IdSet), - vertexRemoves: make(map[ShardAddress]*IdSet), - hyperedgeAdds: make(map[ShardAddress]*IdSet), - hyperedgeRemoves: make(map[ShardAddress]*IdSet), + size: new(big.Int), + vertexAdds: make(map[ShardKey]*IdSet), + vertexRemoves: make(map[ShardKey]*IdSet), + hyperedgeAdds: make(map[ShardKey]*IdSet), + hyperedgeRemoves: make(map[ShardKey]*IdSet), } } -func (hg *Hypergraph) getOrCreateIdSet(shardAddr ShardAddress, addMap map[ShardAddress]*IdSet, removeMap map[ShardAddress]*IdSet, atomType AtomType) (*IdSet, *IdSet) { +func (hg *Hypergraph) GetVertexAdds() map[ShardKey]*IdSet { + return hg.vertexAdds +} + +func (hg *Hypergraph) GetVertexRemoves() map[ShardKey]*IdSet { + return hg.vertexRemoves +} + +func (hg *Hypergraph) GetHyperedgeAdds() map[ShardKey]*IdSet { + return hg.hyperedgeAdds +} + +func (hg *Hypergraph) GetHyperedgeRemoves() map[ShardKey]*IdSet { + return hg.hyperedgeRemoves +} + +func (hg *Hypergraph) Commit() [][]byte { + commits := [][]byte{} + for _, vertexAdds := range hg.vertexAdds { + commits = append(commits, vertexAdds.tree.Commit(false)) + } + for _, vertexRemoves := range hg.vertexRemoves { + commits = append(commits, vertexRemoves.tree.Commit(false)) + } + for _, hyperedgeAdds := range hg.hyperedgeAdds { + commits = append(commits, hyperedgeAdds.tree.Commit(false)) + } + for _, hyperedgeRemoves := range hg.hyperedgeRemoves { + commits = append(commits, hyperedgeRemoves.tree.Commit(false)) + } + return commits +} + +func (hg *Hypergraph) ImportFromBytes( + atomType AtomType, + phaseType PhaseType, + shardKey ShardKey, + data []byte, +) error { + set := NewIdSet(atomType) + if err := set.FromBytes(data); err != nil { + return errors.Wrap(err, "import from bytes") + } + + switch atomType { + case VertexAtomType: + switch phaseType { + case AddsPhaseType: + hg.size.Add(hg.size, set.GetSize()) + hg.vertexAdds[shardKey] = set + case RemovesPhaseType: + hg.size.Sub(hg.size, set.GetSize()) + hg.vertexRemoves[shardKey] = set + } + case HyperedgeAtomType: + switch phaseType { + case AddsPhaseType: + hg.size.Add(hg.size, set.GetSize()) + hg.hyperedgeAdds[shardKey] = set + case RemovesPhaseType: + hg.size.Sub(hg.size, set.GetSize()) + hg.hyperedgeRemoves[shardKey] = set + } + } + + return nil +} + +func (hg *Hypergraph) GetSize() *big.Int { + return hg.size +} + +func (hg *Hypergraph) getOrCreateIdSet( + shardAddr ShardKey, + addMap map[ShardKey]*IdSet, + removeMap map[ShardKey]*IdSet, + atomType AtomType, +) (*IdSet, *IdSet) { if _, ok := addMap[shardAddr]; !ok { addMap[shardAddr] = NewIdSet(atomType) } @@ -165,88 +502,149 @@ func (hg *Hypergraph) getOrCreateIdSet(shardAddr ShardAddress, addMap map[ShardA return addMap[shardAddr], removeMap[shardAddr] } -func (hg *Hypergraph) AddVertex(v *Vertex) error { - shardAddr := GetShardAddress(v) - addSet, _ := hg.getOrCreateIdSet(shardAddr, hg.vertexAdds, hg.vertexRemoves, "vertex") - return addSet.Add(v) +func (hg *Hypergraph) AddVertex(v Vertex) error { + shardAddr := GetShardKey(v) + addSet, _ := hg.getOrCreateIdSet( + shardAddr, + hg.vertexAdds, + hg.vertexRemoves, + VertexAtomType, + ) + hg.size.Add(hg.size, v.GetSize()) + return errors.Wrap(addSet.Add(v), "add vertex") } -func (hg *Hypergraph) AddHyperedge(h *Hyperedge) error { - if !hg.LookupAtomSet(h.Extrinsics) { +func (hg *Hypergraph) AddHyperedge(h Hyperedge) error { + if !hg.LookupAtomSet(&h.(*hyperedge).extrinsics) { return ErrMissingExtrinsics } - shardAddr := GetShardAddress(h) - addSet, _ := hg.getOrCreateIdSet(shardAddr, hg.hyperedgeAdds, hg.hyperedgeRemoves, "hyperedge") - return addSet.Add(h) + shardAddr := GetShardKey(h) + addSet, removeSet := hg.getOrCreateIdSet( + shardAddr, + hg.hyperedgeAdds, + hg.hyperedgeRemoves, + HyperedgeAtomType, + ) + id := h.GetID() + if !removeSet.Has(id) { + hg.size.Add(hg.size, h.GetSize()) + return errors.Wrap(addSet.Add(h), "add hyperedge") + } + return nil } -func (hg *Hypergraph) RemoveVertex(v *Vertex) error { - shardAddr := GetShardAddress(v) - - if !hg.LookupVertex(v) { - _, removeSet := hg.getOrCreateIdSet(shardAddr, hg.vertexAdds, hg.vertexRemoves, "vertex") - return removeSet.Add(v) +func (hg *Hypergraph) RemoveVertex(v Vertex) error { + shardKey := GetShardKey(v) + if !hg.LookupVertex(v.(*vertex)) { + addSet, removeSet := hg.getOrCreateIdSet( + shardKey, + hg.vertexAdds, + hg.vertexRemoves, + VertexAtomType, + ) + if err := addSet.Add(v); err != nil { + return errors.Wrap(err, "remove vertex") + } + return errors.Wrap(removeSet.Add(v), "remove vertex") } + id := v.GetID() + for _, hyperedgeAdds := range hg.hyperedgeAdds { for _, atom := range hyperedgeAdds.atoms { - if he, ok := atom.(*Hyperedge); ok { - if _, ok := he.Extrinsics[v.GetID()]; ok { + if he, ok := atom.(*hyperedge); ok { + if _, ok := he.extrinsics[id]; ok { return ErrIsExtrinsic } } } } - _, removeSet := hg.getOrCreateIdSet(shardAddr, hg.vertexAdds, hg.vertexRemoves, "vertex") - return removeSet.Add(v) + _, removeSet := hg.getOrCreateIdSet( + shardKey, + hg.vertexAdds, + hg.vertexRemoves, + VertexAtomType, + ) + hg.size.Sub(hg.size, v.GetSize()) + err := removeSet.Add(v) + return err } -func (hg *Hypergraph) RemoveHyperedge(h *Hyperedge) error { - shardAddr := GetShardAddress(h) +func (hg *Hypergraph) RemoveHyperedge(h Hyperedge) error { + shardKey := GetShardKey(h) + wasPresent := hg.LookupHyperedge(h.(*hyperedge)) + if !wasPresent { + addSet, removeSet := hg.getOrCreateIdSet( + shardKey, + hg.hyperedgeAdds, + hg.hyperedgeRemoves, + HyperedgeAtomType, + ) + if err := addSet.Add(h); err != nil { + return errors.Wrap(err, "remove hyperedge") + } - if !hg.LookupHyperedge(h) { - _, removeSet := hg.getOrCreateIdSet(shardAddr, hg.hyperedgeAdds, hg.hyperedgeRemoves, "hyperedge") - return removeSet.Add(h) + return errors.Wrap(removeSet.Add(h), "remove hyperedge") } + id := h.GetID() for _, hyperedgeAdds := range hg.hyperedgeAdds { for _, atom := range hyperedgeAdds.atoms { - if he, ok := atom.(*Hyperedge); ok { - if _, ok := he.Extrinsics[h.GetID()]; ok { + if he, ok := atom.(*hyperedge); ok { + if _, ok := he.extrinsics[id]; ok { return ErrIsExtrinsic } } } } - _, removeSet := hg.getOrCreateIdSet(shardAddr, hg.hyperedgeAdds, hg.hyperedgeRemoves, "hyperedge") - return removeSet.Add(h) + _, removeSet := hg.getOrCreateIdSet( + shardKey, + hg.hyperedgeAdds, + hg.hyperedgeRemoves, + HyperedgeAtomType, + ) + hg.size.Sub(hg.size, h.GetSize()) + err := removeSet.Add(h) + return err } -func (hg *Hypergraph) LookupVertex(v *Vertex) bool { - shardAddr := GetShardAddress(v) - addSet, removeSet := hg.getOrCreateIdSet(shardAddr, hg.vertexAdds, hg.vertexRemoves, "vertex") - return addSet.Has(v) && !removeSet.Has(v) +func (hg *Hypergraph) LookupVertex(v Vertex) bool { + shardAddr := GetShardKey(v) + addSet, removeSet := hg.getOrCreateIdSet( + shardAddr, + hg.vertexAdds, + hg.vertexRemoves, + VertexAtomType, + ) + id := v.GetID() + return addSet.Has(id) && !removeSet.Has(id) } -func (hg *Hypergraph) LookupHyperedge(h *Hyperedge) bool { - shardAddr := GetShardAddress(h) - addSet, removeSet := hg.getOrCreateIdSet(shardAddr, hg.hyperedgeAdds, hg.hyperedgeRemoves, "hyperedge") - return hg.LookupAtomSet(h.Extrinsics) && addSet.Has(h) && !removeSet.Has(h) +func (hg *Hypergraph) LookupHyperedge(h Hyperedge) bool { + shardAddr := GetShardKey(h) + addSet, removeSet := hg.getOrCreateIdSet( + shardAddr, + hg.hyperedgeAdds, + hg.hyperedgeRemoves, + HyperedgeAtomType, + ) + id := h.GetID() + return hg.LookupAtomSet(&h.(*hyperedge).extrinsics) && addSet.Has(id) && !removeSet.Has(id) } func (hg *Hypergraph) LookupAtom(a Atom) bool { switch v := a.(type) { - case *Vertex: + case *vertex: return hg.LookupVertex(v) - case *Hyperedge: + case *hyperedge: return hg.LookupHyperedge(v) default: return false } } -func (hg *Hypergraph) LookupAtomSet(atomSet map[[64]byte]Atom) bool { - for _, atom := range atomSet { +func (hg *Hypergraph) LookupAtomSet(atomSet *map[[64]byte]Atom) bool { + for _, atom := range *atomSet { if !hg.LookupAtom(atom) { return false } @@ -255,12 +653,13 @@ func (hg *Hypergraph) LookupAtomSet(atomSet map[[64]byte]Atom) bool { } func (hg *Hypergraph) Within(a, h Atom) bool { - if he, ok := h.(*Hyperedge); ok { - if _, ok := he.Extrinsics[a.GetID()]; ok || a.GetID() == h.GetID() { + if he, ok := h.(*hyperedge); ok { + addr := a.GetID() + if _, ok := he.extrinsics[addr]; ok || a.GetID() == h.GetID() { return true } - for _, extrinsic := range he.Extrinsics { - if nestedHe, ok := extrinsic.(*Hyperedge); ok { + for _, extrinsic := range he.extrinsics { + if nestedHe, ok := extrinsic.(*hyperedge); ok { if hg.LookupHyperedge(nestedHe) && hg.Within(a, nestedHe) { return true } @@ -270,15 +669,15 @@ func (hg *Hypergraph) Within(a, h Atom) bool { return false } -// GetReconciledVertexSetForShard computes the set of vertices that have been added but -// not removed for a specific shard. -func (hg *Hypergraph) GetReconciledVertexSetForShard(shardAddr ShardAddress) *IdSet { - vertices := NewIdSet("vertex") +func (hg *Hypergraph) GetReconciledVertexSetForShard( + shardKey ShardKey, +) *IdSet { + vertices := NewIdSet(VertexAtomType) - if addSet, ok := hg.vertexAdds[shardAddr]; ok { - removeSet := hg.vertexRemoves[shardAddr] - for _, v := range addSet.atoms { - if !removeSet.Has(v) { + if addSet, ok := hg.vertexAdds[shardKey]; ok { + removeSet := hg.vertexRemoves[shardKey] + for id, v := range addSet.atoms { + if !removeSet.Has(id) { vertices.Add(v) } } @@ -287,15 +686,15 @@ func (hg *Hypergraph) GetReconciledVertexSetForShard(shardAddr ShardAddress) *Id return vertices } -// GetReconciledHyperedgeSetForShard computes the set of hyperedges that have been added -// but not removed for a specific shard. -func (hg *Hypergraph) GetReconciledHyperedgeSetForShard(shardAddr ShardAddress) *IdSet { - hyperedges := NewIdSet("hyperedge") +func (hg *Hypergraph) GetReconciledHyperedgeSetForShard( + shardKey ShardKey, +) *IdSet { + hyperedges := NewIdSet(HyperedgeAtomType) - if addSet, ok := hg.hyperedgeAdds[shardAddr]; ok { - removeSet := hg.hyperedgeRemoves[shardAddr] + if addSet, ok := hg.hyperedgeAdds[shardKey]; ok { + removeSet := hg.hyperedgeRemoves[shardKey] for _, h := range addSet.atoms { - if !removeSet.Has(h) { + if !removeSet.Has(h.GetID()) { hyperedges.Add(h) } } diff --git a/node/hypergraph/application/hypergraph_convergence_test.go b/node/hypergraph/application/hypergraph_convergence_test.go index 81a7c55..1621f1e 100644 --- a/node/hypergraph/application/hypergraph_convergence_test.go +++ b/node/hypergraph/application/hypergraph_convergence_test.go @@ -1,56 +1,59 @@ package application_test import ( + crand "crypto/rand" + "fmt" "math/rand" "testing" "time" + "github.com/cloudflare/circl/sign/ed448" + "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" ) type Operation struct { Type string // "AddVertex", "RemoveVertex", "AddHyperedge", "RemoveHyperedge" - Vertex *application.Vertex - Hyperedge *application.Hyperedge + Vertex application.Vertex + Hyperedge application.Hyperedge } func TestConvergence(t *testing.T) { - numParties := 3 - numOperations := 100 - - // Generate a set of vertices and hyperedges - vertices := make([]*application.Vertex, numOperations) + numParties := 4 + numOperations := 100000 + enc := crypto.NewMPCitHVerifiableEncryptor(1) + pub, _, _ := ed448.GenerateKey(crand.Reader) + enc.Encrypt(make([]byte, 20), pub) + vertices := make([]application.Vertex, numOperations) for i := 0; i < numOperations; i++ { - vertices[i] = &application.Vertex{ - AppAddress: [32]byte{byte(i % 256)}, - DataAddress: [32]byte{byte(i / 256)}, - } + vertices[i] = application.NewVertex( + [32]byte{byte((i >> 8) % 256), byte((i % 256))}, + [32]byte{byte((i >> 8) / 256), byte(i / 256)}, + []application.Encrypted{}, + ) } - hyperedges := make([]*application.Hyperedge, numOperations/10) + hyperedges := make([]application.Hyperedge, numOperations/10) for i := 0; i < numOperations/10; i++ { - hyperedges[i] = &application.Hyperedge{ - AppAddress: [32]byte{byte(i % 256)}, - DataAddress: [32]byte{byte(i / 256)}, - Extrinsics: make(map[[64]byte]application.Atom), - } - // Add some random vertices as extrinsics + hyperedges[i] = application.NewHyperedge( + [32]byte{0, 0, byte((i >> 8) % 256), byte(i % 256)}, + [32]byte{0, 0, byte((i >> 8) / 256), byte(i / 256)}, + ) for j := 0; j < 3; j++ { v := vertices[rand.Intn(len(vertices))] - hyperedges[i].Extrinsics[v.GetID()] = v + hyperedges[i].AddExtrinsic(v) } } - // Generate a sequence of operations operations1 := make([]Operation, numOperations) operations2 := make([]Operation, numOperations) for i := 0; i < numOperations; i++ { op := rand.Intn(2) switch op { case 0: - operations1[i] = Operation{Type: "AddVertex", Vertex: vertices[rand.Intn(len(vertices))]} + operations1[i] = Operation{Type: "AddVertex", Vertex: vertices[i]} case 1: - operations1[i] = Operation{Type: "RemoveVertex", Vertex: vertices[rand.Intn(len(vertices))]} + operations1[i] = Operation{Type: "AddVertex", Vertex: vertices[i]} } } for i := 0; i < numOperations; i++ { @@ -63,13 +66,11 @@ func TestConvergence(t *testing.T) { } } - // Create CRDTs for each party crdts := make([]*application.Hypergraph, numParties) for i := 0; i < numParties; i++ { crdts[i] = application.NewHypergraph() } - // Apply operations in different orders for each party for i := 0; i < numParties; i++ { rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(operations1), func(i, j int) { operations1[i], operations1[j] = operations1[j], operations1[i] }) @@ -94,15 +95,17 @@ func TestConvergence(t *testing.T) { case "RemoveVertex": crdts[i].RemoveVertex(op.Vertex) case "AddHyperedge": + fmt.Println("add", i, op) crdts[i].AddHyperedge(op.Hyperedge) case "RemoveHyperedge": + fmt.Println("remove", i, op) crdts[i].RemoveHyperedge(op.Hyperedge) } } } - // Verify that all CRDTs have converged to the same state - // Additional verification: check specific vertices and hyperedges + crdts[0].GetSize() + for _, v := range vertices { state := crdts[0].LookupVertex(v) for i := 1; i < numParties; i++ { @@ -111,12 +114,11 @@ func TestConvergence(t *testing.T) { } } } - for _, h := range hyperedges { state := crdts[0].LookupHyperedge(h) for i := 1; i < numParties; i++ { if crdts[i].LookupHyperedge(h) != state { - t.Errorf("Hyperedge %v has different state in CRDT %d", h, i) + t.Errorf("Hyperedge %v has different state in CRDT %d, %v", h, i, state) } } } diff --git a/node/hypergraph/application/hypergraph_test.go b/node/hypergraph/application/hypergraph_test.go index 25f490d..8d0d1c9 100644 --- a/node/hypergraph/application/hypergraph_test.go +++ b/node/hypergraph/application/hypergraph_test.go @@ -2,6 +2,7 @@ package application_test import ( "bytes" + "fmt" "testing" "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" @@ -12,8 +13,8 @@ func TestHypergraph(t *testing.T) { // Test vertex operations t.Run("Vertex Operations", func(t *testing.T) { - v1 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{1}} - v2 := &application.Vertex{AppAddress: [32]byte{1}, DataAddress: [32]byte{2}} + v1 := application.NewVertex([32]byte{1}, [32]byte{1}, []application.Encrypted{}) + v2 := application.NewVertex([32]byte{1}, [32]byte{2}, []application.Encrypted{}) // Add vertices err := hg.AddVertex(v1) @@ -48,16 +49,14 @@ func TestHypergraph(t *testing.T) { // Test hyperedge operations t.Run("Hyperedge Operations", func(t *testing.T) { - v3 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{1}} - v4 := &application.Vertex{AppAddress: [32]byte{2}, DataAddress: [32]byte{2}} + v3 := application.NewVertex([32]byte{2}, [32]byte{1}, []application.Encrypted{}) + v4 := application.NewVertex([32]byte{2}, [32]byte{2}, []application.Encrypted{}) hg.AddVertex(v3) hg.AddVertex(v4) - h1 := &application.Hyperedge{ - AppAddress: [32]byte{3}, - DataAddress: [32]byte{1}, - Extrinsics: map[[64]byte]application.Atom{v3.GetID(): v3, v4.GetID(): v4}, - } + h1 := application.NewHyperedge([32]byte{3}, [32]byte{1}) + h1.AddExtrinsic(v3) + h1.AddExtrinsic(v4) // Add hyperedge err := hg.AddHyperedge(h1) @@ -82,16 +81,14 @@ func TestHypergraph(t *testing.T) { // Test "within" relationship t.Run("Within Relationship", func(t *testing.T) { - v5 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{1}} - v6 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{2}} + v5 := application.NewVertex([32]byte{4}, [32]byte{1}, []application.Encrypted{}) + v6 := application.NewVertex([32]byte{4}, [32]byte{2}, []application.Encrypted{}) hg.AddVertex(v5) hg.AddVertex(v6) - h2 := &application.Hyperedge{ - AppAddress: [32]byte{5}, - DataAddress: [32]byte{1}, - Extrinsics: map[[64]byte]application.Atom{v5.GetID(): v5, v6.GetID(): v6}, - } + h2 := application.NewHyperedge([32]byte{5}, [32]byte{1}) + h2.AddExtrinsic(v5) + h2.AddExtrinsic(v6) hg.AddHyperedge(h2) if !hg.Within(v5, h2) { @@ -101,7 +98,7 @@ func TestHypergraph(t *testing.T) { t.Error("v6 should be within h2") } - v7 := &application.Vertex{AppAddress: [32]byte{4}, DataAddress: [32]byte{3}} + v7 := application.NewVertex([32]byte{4}, [32]byte{3}, []application.Encrypted{}) hg.AddVertex(v7) if hg.Within(v7, h2) { t.Error("v7 should not be within h2") @@ -110,21 +107,16 @@ func TestHypergraph(t *testing.T) { // Test nested hyperedges t.Run("Nested Hyperedges", func(t *testing.T) { - v8 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{1}} - v9 := &application.Vertex{AppAddress: [32]byte{6}, DataAddress: [32]byte{2}} + v8 := application.NewVertex([32]byte{6}, [32]byte{1}, []application.Encrypted{}) + v9 := application.NewVertex([32]byte{6}, [32]byte{2}, []application.Encrypted{}) hg.AddVertex(v8) hg.AddVertex(v9) - h3 := &application.Hyperedge{ - AppAddress: [32]byte{7}, - DataAddress: [32]byte{1}, - Extrinsics: map[[64]byte]application.Atom{v8.GetID(): v8}, - } - h4 := &application.Hyperedge{ - AppAddress: [32]byte{7}, - DataAddress: [32]byte{2}, - Extrinsics: map[[64]byte]application.Atom{h3.GetID(): h3, v9.GetID(): v9}, - } + h3 := application.NewHyperedge([32]byte{7}, [32]byte{1}) + h3.AddExtrinsic(v8) + h4 := application.NewHyperedge([32]byte{7}, [32]byte{2}) + h4.AddExtrinsic(h3) + h4.AddExtrinsic(v9) hg.AddHyperedge(h3) hg.AddHyperedge(h4) @@ -138,12 +130,10 @@ func TestHypergraph(t *testing.T) { // Test error cases t.Run("Error Cases", func(t *testing.T) { - v10 := &application.Vertex{AppAddress: [32]byte{8}, DataAddress: [32]byte{1}} - h5 := &application.Hyperedge{ - AppAddress: [32]byte{8}, - DataAddress: [32]byte{2}, - Extrinsics: map[[64]byte]application.Atom{v10.GetID(): v10}, - } + v10 := application.NewVertex([32]byte{8}, [32]byte{1}, []application.Encrypted{}) + + h5 := application.NewHyperedge([32]byte{8}, [32]byte{2}) + h5.AddExtrinsic(v10) // Try to add hyperedge with non-existent vertex err := hg.AddHyperedge(h5) @@ -153,8 +143,8 @@ func TestHypergraph(t *testing.T) { // Add vertex and hyperedge hg.AddVertex(v10) + fmt.Println("add hyperedge") hg.AddHyperedge(h5) - // Try to remove vertex that is an extrinsic err = hg.RemoveVertex(v10) if err != application.ErrIsExtrinsic { @@ -164,8 +154,8 @@ func TestHypergraph(t *testing.T) { // Test sharding t.Run("Sharding", func(t *testing.T) { - v11 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{1}} - v12 := &application.Vertex{AppAddress: [32]byte{9}, DataAddress: [32]byte{2}} + v11 := application.NewVertex([32]byte{9}, [32]byte{1}, []application.Encrypted{}) + v12 := application.NewVertex([32]byte{9}, [32]byte{2}, []application.Encrypted{}) hg.AddVertex(v11) hg.AddVertex(v12) @@ -173,8 +163,9 @@ func TestHypergraph(t *testing.T) { shard12 := application.GetShardAddress(v12) if !bytes.Equal(shard11.L1[:], shard12.L1[:]) || - bytes.Equal(shard11.L2[:], shard12.L2[:]) { - t.Error("v11 and v12 should be in the same L1 shard and not the same L2 shard") + !bytes.Equal(shard11.L2[:], shard12.L2[:]) || + bytes.Equal(shard11.L3[:], shard12.L3[:]) { + t.Error("v11 and v12 should be in the same L1 shard and the same L2 shard but not the same L3 shard") } }) } diff --git a/node/main.go b/node/main.go index eca3a16..56512f5 100644 --- a/node/main.go +++ b/node/main.go @@ -7,7 +7,6 @@ import ( _ "embed" "encoding/binary" "encoding/hex" - "encoding/json" "flag" "fmt" "io/fs" @@ -22,14 +21,12 @@ import ( "runtime" rdebug "runtime/debug" "runtime/pprof" - "slices" "strconv" "strings" "syscall" "time" "github.com/cloudflare/circl/sign/ed448" - "github.com/iden3/go-iden3-crypto/poseidon" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/pbnjay/memory" @@ -42,14 +39,10 @@ import ( "source.quilibrium.com/quilibrium/monorepo/node/config" qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/crypto/kzg" - "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token" - "source.quilibrium.com/quilibrium/monorepo/node/execution/intrinsics/token/application" qruntime "source.quilibrium.com/quilibrium/monorepo/node/internal/runtime" - "source.quilibrium.com/quilibrium/monorepo/node/p2p" "source.quilibrium.com/quilibrium/monorepo/node/protobufs" "source.quilibrium.com/quilibrium/monorepo/node/rpc" "source.quilibrium.com/quilibrium/monorepo/node/store" - "source.quilibrium.com/quilibrium/monorepo/node/tries" "source.quilibrium.com/quilibrium/monorepo/node/utils" ) @@ -497,8 +490,6 @@ func main() { } } - RunForkRepairIfNeeded(nodeConfig) - done := make(chan os.Signal, 1) signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) var node *app.Node @@ -605,551 +596,6 @@ func stopDataWorkers() { } } -//go:embed overrideFrames.json -var overrideFramesData []byte - -func RunForkRepairIfNeeded( - nodeConfig *config.Config, -) { - logger, _ := zap.NewDevelopment() - db := store.NewPebbleDB(&config.DBConfig{Path: nodeConfig.DB.Path}) - defer db.Close() - clockStore := store.NewPebbleClockStore(db, logger) - coinStore := store.NewPebbleCoinStore(db, logger) - filter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - frame, _, err := clockStore.GetDataClockFrame(filter, uint64(48995), false) - if err != nil { - fmt.Println("No repair needed.") - return - } - - compareSel, _ := frame.GetSelector() - badFrameSelector, _ := hex.DecodeString("16515bf99a55d24c35d1dd0a0c7d778154e5ffa6dfa3ad164f11355f4cb00056") - - if bytes.Equal(badFrameSelector, compareSel.FillBytes(make([]byte, 32))) { - logger.Info("performing fork repair") - txn, _ := coinStore.NewTransaction(false) - _, outs, _ := application.GetOutputsFromClockFrame(frame) - logger.Info("removing invalid frame at position 48995") - for i, output := range outs.Outputs { - switch o := output.Output.(type) { - case *protobufs.TokenOutput_Coin: - address, _ := token.GetAddressOfCoin(o.Coin, frame.FrameNumber, uint64(i)) - coin, err := coinStore.GetCoinByAddress(nil, address) - if err != nil { - fmt.Println(err) - return - } - stateTree := &qcrypto.VectorCommitmentTree{} - if err = coinStore.DeleteCoin(txn, address, coin, stateTree); err != nil { - txn.Abort() - fmt.Println(err) - return - } - case *protobufs.TokenOutput_Proof: - address, _ := token.GetAddressOfPreCoinProof(o.Proof) - proof, err := coinStore.GetPreCoinProofByAddress(address) - if err != nil { - txn.Abort() - fmt.Println(err) - return - } - stateTree := &qcrypto.VectorCommitmentTree{} - if err = coinStore.DeletePreCoinProof(txn, address, proof, stateTree); err != nil { - txn.Abort() - fmt.Println(err) - return - } - } - } - - if err = txn.Commit(); err != nil { - txn.Abort() - - logger.Error("could not commit data", zap.Error(err)) - return - } - - logger.Info("inserting valid frame starting at position 48995") - type OverrideFrames struct { - FrameData []byte `json:"frameData"` - } - overrideFramesJson := []*OverrideFrames{} - if err = json.Unmarshal(overrideFramesData, &overrideFramesJson); err != nil { - txn.Abort() - logger.Error("could not unmarshal overriding frame data", zap.Error(err)) - return - } - - for _, overrideFrame := range overrideFramesJson { - override := &protobufs.ClockFrame{} - if err := proto.Unmarshal(overrideFrame.FrameData, override); err != nil { - logger.Error("could not unmarshal frame data", zap.Error(err)) - return - } - - txn, _ := clockStore.NewTransaction(false) - if err := overrideHead( - txn, - clockStore, - coinStore, - override, - logger, - ); err != nil { - txn.Abort() - logger.Error("could not override frame data", zap.Error(err)) - return - } - - if err = txn.Commit(); err != nil { - txn.Abort() - - logger.Error("could not commit data", zap.Error(err)) - return - } - } - } else { - fmt.Println("No repair needed.") - return - } -} - -func overrideHead( - txn store.Transaction, - clockStore store.ClockStore, - coinStore store.CoinStore, - frame *protobufs.ClockFrame, - logger *zap.Logger, -) error { - selector, err := frame.GetSelector() - if err != nil { - panic(err) - } - filter := p2p.GetBloomFilter(application.TOKEN_ADDRESS, 256, 3) - - _, ts, err := clockStore.GetDataClockFrame( - filter, - frame.FrameNumber-1, - false, - ) - if err != nil { - logger.Error("could not get frame", zap.Error(err), zap.Uint64("frame", frame.FrameNumber-1)) - return errors.Wrap(err, "set head") - } - - if err := clockStore.StageDataClockFrame( - selector.FillBytes(make([]byte, 32)), - frame, - txn, - ); err != nil { - panic(err) - } - - if ts, err = processFrame(txn, frame, ts, coinStore, clockStore, logger); err != nil { - logger.Error("invalid frame execution, unwinding", zap.Error(err)) - txn.Abort() - return errors.Wrap(err, "set head") - } - - if err := clockStore.CommitDataClockFrame( - filter, - frame.FrameNumber, - selector.FillBytes(make([]byte, 32)), - ts, - txn, - false, - ); err != nil { - panic(err) - } - - return nil -} - -func processFrame( - txn store.Transaction, - frame *protobufs.ClockFrame, - triesAtFrame []*tries.RollingFrecencyCritbitTrie, - coinStore store.CoinStore, - clockStore store.ClockStore, - logger *zap.Logger, -) ([]*tries.RollingFrecencyCritbitTrie, error) { - f, err := coinStore.GetLatestFrameProcessed() - if err != nil || f == frame.FrameNumber { - return nil, errors.Wrap(err, "process frame") - } - - logger.Info( - "evaluating next frame", - zap.Uint64( - "frame_number", - frame.FrameNumber, - ), - ) - m, err := clockStore.GetPeerSeniorityMap(frame.Filter) - if err != nil { - logger.Error( - "error while materializing seniority map", - zap.Error(err), - ) - return nil, errors.Wrap(err, "process frame") - } - peerSeniority := token.NewFromMap(m) - - app, err := application.MaterializeApplicationFromFrame( - nil, - frame, - triesAtFrame, - coinStore, - clockStore, - nil, - logger, - nil, - ) - if err != nil { - logger.Error( - "error while materializing application from frame", - zap.Error(err), - ) - return nil, errors.Wrap(err, "process frame") - } - - proverTrieJoinRequests := [][]byte{} - proverTrieLeaveRequests := [][]byte{} - - for i, output := range app.TokenOutputs.Outputs { - i := i - if frame.FrameNumber == 0 { - i = 0 - } - switch o := output.Output.(type) { - case *protobufs.TokenOutput_Coin: - address, err := token.GetAddressOfCoin(o.Coin, frame.FrameNumber, uint64(i)) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - stateTree := &qcrypto.VectorCommitmentTree{} - err = coinStore.PutCoin( - txn, - frame.FrameNumber, - address, - o.Coin, - stateTree, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - case *protobufs.TokenOutput_DeletedCoin: - coin, err := coinStore.GetCoinByAddress(txn, o.DeletedCoin.Address) - if err != nil { - if frame.FrameNumber == 48997 { - // special case, the fork happened at 48995, state replayed here - continue - } - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - stateTree := &qcrypto.VectorCommitmentTree{} - err = coinStore.DeleteCoin( - txn, - o.DeletedCoin.Address, - coin, - stateTree, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - case *protobufs.TokenOutput_Proof: - address, err := token.GetAddressOfPreCoinProof(o.Proof) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - stateTree := &qcrypto.VectorCommitmentTree{} - err = coinStore.PutPreCoinProof( - txn, - frame.FrameNumber, - address, - o.Proof, - stateTree, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - if len(o.Proof.Amount) == 32 && - !bytes.Equal(o.Proof.Amount, make([]byte, 32)) && - o.Proof.Commitment != nil { - addr := string(o.Proof.Owner.GetImplicitAccount().Address) - for _, t := range app.Tries { - if t.Contains([]byte(addr)) { - t.Add([]byte(addr), frame.FrameNumber) - break - } - } - if _, ok := (*peerSeniority)[addr]; !ok { - (*peerSeniority)[addr] = token.NewPeerSeniorityItem(10, addr) - } else { - (*peerSeniority)[addr] = token.NewPeerSeniorityItem( - (*peerSeniority)[addr].GetSeniority()+10, - addr, - ) - } - } - case *protobufs.TokenOutput_DeletedProof: - address, err := token.GetAddressOfPreCoinProof(o.DeletedProof) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - stateTree := &qcrypto.VectorCommitmentTree{} - err = coinStore.DeletePreCoinProof( - txn, - address, - o.DeletedProof, - stateTree, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - case *protobufs.TokenOutput_Announce: - peerIds := []string{} - for _, sig := range o.Announce.PublicKeySignaturesEd448 { - peerId, err := getPeerIdFromSignature(sig) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - peerIds = append(peerIds, peerId.String()) - } - - mergeable := true - for i, peerId := range peerIds { - addr, err := getAddressFromSignature( - o.Announce.PublicKeySignaturesEd448[i], - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - sen, ok := (*peerSeniority)[string(addr)] - if !ok { - continue - } - - peer := new(big.Int).SetUint64(sen.GetSeniority()) - if peer.Cmp(token.GetAggregatedSeniority([]string{peerId})) != 0 { - mergeable = false - break - } - } - - if mergeable { - addr, err := getAddressFromSignature( - o.Announce.PublicKeySignaturesEd448[0], - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - additional := uint64(0) - _, prfs, err := coinStore.GetPreCoinProofsForOwner(addr) - if err != nil && !errors.Is(err, store.ErrNotFound) { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - for _, pr := range prfs { - if pr.IndexProof == nil && pr.Difficulty == 0 && pr.Commitment == nil { - // approximate average per interval: - add := new(big.Int).SetBytes(pr.Amount) - add.Quo(add, big.NewInt(58800000)) - if add.Cmp(big.NewInt(4000000)) > 0 { - add = big.NewInt(4000000) - } - additional = add.Uint64() - } - } - - (*peerSeniority)[string(addr)] = token.NewPeerSeniorityItem( - token.GetAggregatedSeniority(peerIds).Uint64()+additional, - string(addr), - ) - - for _, sig := range o.Announce.PublicKeySignaturesEd448[1:] { - addr, err := getAddressFromSignature( - sig, - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - (*peerSeniority)[string(addr)] = token.NewPeerSeniorityItem(0, string(addr)) - } - } - case *protobufs.TokenOutput_Join: - addr, err := getAddressFromSignature(o.Join.PublicKeySignatureEd448) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - if _, ok := (*peerSeniority)[string(addr)]; !ok { - (*peerSeniority)[string(addr)] = token.NewPeerSeniorityItem(20, string(addr)) - } else { - (*peerSeniority)[string(addr)] = token.NewPeerSeniorityItem( - (*peerSeniority)[string(addr)].GetSeniority()+20, - string(addr), - ) - } - proverTrieJoinRequests = append(proverTrieJoinRequests, addr) - case *protobufs.TokenOutput_Leave: - addr, err := getAddressFromSignature(o.Leave.PublicKeySignatureEd448) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - proverTrieLeaveRequests = append(proverTrieLeaveRequests, addr) - case *protobufs.TokenOutput_Pause: - _, err := getAddressFromSignature(o.Pause.PublicKeySignatureEd448) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - case *protobufs.TokenOutput_Resume: - _, err := getAddressFromSignature(o.Resume.PublicKeySignatureEd448) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - case *protobufs.TokenOutput_Penalty: - addr := string(o.Penalty.Account.GetImplicitAccount().Address) - if _, ok := (*peerSeniority)[addr]; !ok { - (*peerSeniority)[addr] = token.NewPeerSeniorityItem(0, addr) - proverTrieLeaveRequests = append(proverTrieLeaveRequests, []byte(addr)) - } else { - if (*peerSeniority)[addr].GetSeniority() > o.Penalty.Quantity { - for _, t := range app.Tries { - if t.Contains([]byte(addr)) { - v := t.Get([]byte(addr)) - latest := v.LatestFrame - if frame.FrameNumber-latest > 100 { - proverTrieLeaveRequests = append(proverTrieLeaveRequests, []byte(addr)) - } - break - } - } - (*peerSeniority)[addr] = token.NewPeerSeniorityItem( - (*peerSeniority)[addr].GetSeniority()-o.Penalty.Quantity, - addr, - ) - } else { - (*peerSeniority)[addr] = token.NewPeerSeniorityItem(0, addr) - proverTrieLeaveRequests = append(proverTrieLeaveRequests, []byte(addr)) - } - } - } - } - - joinAddrs := tries.NewMinHeap[token.PeerSeniorityItem]() - leaveAddrs := tries.NewMinHeap[token.PeerSeniorityItem]() - for _, addr := range proverTrieJoinRequests { - if _, ok := (*peerSeniority)[string(addr)]; !ok { - joinAddrs.Push(token.NewPeerSeniorityItem(0, string(addr))) - } else { - joinAddrs.Push((*peerSeniority)[string(addr)]) - } - } - for _, addr := range proverTrieLeaveRequests { - if _, ok := (*peerSeniority)[string(addr)]; !ok { - leaveAddrs.Push(token.NewPeerSeniorityItem(0, string(addr))) - } else { - leaveAddrs.Push((*peerSeniority)[string(addr)]) - } - } - - joinReqs := make([]token.PeerSeniorityItem, len(joinAddrs.All())) - copy(joinReqs, joinAddrs.All()) - slices.Reverse(joinReqs) - leaveReqs := make([]token.PeerSeniorityItem, len(leaveAddrs.All())) - copy(leaveReqs, leaveAddrs.All()) - slices.Reverse(leaveReqs) - - token.ProcessJoinsAndLeaves(joinReqs, leaveReqs, app, peerSeniority, frame) - - err = clockStore.PutPeerSeniorityMap( - txn, - frame.Filter, - token.ToSerializedMap(peerSeniority), - ) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - err = coinStore.SetLatestFrameProcessed(txn, frame.FrameNumber) - if err != nil { - txn.Abort() - return nil, errors.Wrap(err, "process frame") - } - - return app.Tries, nil -} - -func getPeerIdFromSignature( - sig *protobufs.Ed448Signature, -) (peer.ID, error) { - if sig.PublicKey == nil || sig.PublicKey.KeyValue == nil { - return "", errors.New("invalid data") - } - - pk, err := crypto.UnmarshalEd448PublicKey( - sig.PublicKey.KeyValue, - ) - if err != nil { - return "", errors.Wrap(err, "get address from signature") - } - - peerId, err := peer.IDFromPublicKey(pk) - if err != nil { - return "", errors.Wrap(err, "get address from signature") - } - - return peerId, nil -} - -func getAddressFromSignature( - sig *protobufs.Ed448Signature, -) ([]byte, error) { - if sig.PublicKey == nil || sig.PublicKey.KeyValue == nil { - return nil, errors.New("invalid data") - } - - pk, err := crypto.UnmarshalEd448PublicKey( - sig.PublicKey.KeyValue, - ) - if err != nil { - return nil, errors.Wrap(err, "get address from signature") - } - - peerId, err := peer.IDFromPublicKey(pk) - if err != nil { - return nil, errors.Wrap(err, "get address from signature") - } - - altAddr, err := poseidon.HashBytes([]byte(peerId)) - if err != nil { - return nil, errors.Wrap(err, "get address from signature") - } - - return altAddr.FillBytes(make([]byte, 32)), nil -} - func RunSelfTestIfNeeded( configDir string, nodeConfig *config.Config, diff --git a/node/store/coin.go b/node/store/coin.go index d4e4170..d1bc1a7 100644 --- a/node/store/coin.go +++ b/node/store/coin.go @@ -10,7 +10,6 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" "google.golang.org/protobuf/proto" - "source.quilibrium.com/quilibrium/monorepo/node/crypto" "source.quilibrium.com/quilibrium/monorepo/node/protobufs" ) @@ -31,26 +30,22 @@ type CoinStore interface { frameNumber uint64, address []byte, coin *protobufs.Coin, - stateTree *crypto.VectorCommitmentTree, ) error DeleteCoin( txn Transaction, address []byte, coin *protobufs.Coin, - stateTree *crypto.VectorCommitmentTree, ) error PutPreCoinProof( txn Transaction, frameNumber uint64, address []byte, preCoinProof *protobufs.PreCoinProof, - stateTree *crypto.VectorCommitmentTree, ) error DeletePreCoinProof( txn Transaction, address []byte, preCoinProof *protobufs.PreCoinProof, - stateTree *crypto.VectorCommitmentTree, ) error GetLatestFrameProcessed() (uint64, error) SetLatestFrameProcessed(txn Transaction, frameNumber uint64) error @@ -285,7 +280,6 @@ func (p *PebbleCoinStore) PutCoin( frameNumber uint64, address []byte, coin *protobufs.Coin, - stateTree *crypto.VectorCommitmentTree, ) error { coinBytes, err := proto.Marshal(coin) if err != nil { @@ -311,10 +305,6 @@ func (p *PebbleCoinStore) PutCoin( return errors.Wrap(err, "put coin") } - if err = stateTree.Insert(address, data); err != nil { - return errors.Wrap(err, "put coin") - } - return nil } @@ -322,7 +312,6 @@ func (p *PebbleCoinStore) DeleteCoin( txn Transaction, address []byte, coin *protobufs.Coin, - stateTree *crypto.VectorCommitmentTree, ) error { err := txn.Delete(coinKey(address)) if err != nil { @@ -336,10 +325,6 @@ func (p *PebbleCoinStore) DeleteCoin( return errors.Wrap(err, "delete coin") } - if err = stateTree.Delete(address); err != nil { - return errors.Wrap(err, "delete coin") - } - return nil } @@ -348,7 +333,6 @@ func (p *PebbleCoinStore) PutPreCoinProof( frameNumber uint64, address []byte, preCoinProof *protobufs.PreCoinProof, - stateTree *crypto.VectorCommitmentTree, ) error { proofBytes, err := proto.Marshal(preCoinProof) if err != nil { @@ -374,10 +358,6 @@ func (p *PebbleCoinStore) PutPreCoinProof( return errors.Wrap(err, "put pre coin proof") } - if err = stateTree.Insert(address, data); err != nil { - return errors.Wrap(err, "put pre coin proof") - } - return nil } @@ -385,7 +365,6 @@ func (p *PebbleCoinStore) DeletePreCoinProof( txn Transaction, address []byte, preCoinProof *protobufs.PreCoinProof, - stateTree *crypto.VectorCommitmentTree, ) error { err := txn.Delete(proofKey(address)) if err != nil { @@ -406,10 +385,6 @@ func (p *PebbleCoinStore) DeletePreCoinProof( return errors.Wrap(err, "delete pre coin proof") } - if err = stateTree.Delete(address); err != nil { - return errors.Wrap(err, "delete pre coin proof") - } - return nil } diff --git a/node/store/hypergraph.go b/node/store/hypergraph.go new file mode 100644 index 0000000..aca7c22 --- /dev/null +++ b/node/store/hypergraph.go @@ -0,0 +1,238 @@ +package store + +import ( + "github.com/pkg/errors" + "go.uber.org/zap" + "source.quilibrium.com/quilibrium/monorepo/node/hypergraph/application" +) + +type HypergraphStore interface { + NewTransaction(indexed bool) (Transaction, error) + LoadHypergraph() ( + *application.Hypergraph, + error, + ) + SaveHypergraph( + txn Transaction, + hg *application.Hypergraph, + ) error +} + +var _ HypergraphStore = (*PebbleHypergraphStore)(nil) + +type PebbleHypergraphStore struct { + db KVDB + logger *zap.Logger +} + +func NewPebbleHypergraphStore( + db KVDB, + logger *zap.Logger, +) *PebbleHypergraphStore { + return &PebbleHypergraphStore{ + db, + logger, + } +} + +const ( + HYPERGRAPH_SHARD = 0x09 + VERTEX_ADDS = 0x00 + VERTEX_REMOVES = 0x10 + HYPEREDGE_ADDS = 0x01 + HYPEREDGE_REMOVES = 0x11 +) + +func hypergraphVertexAddsKey(shardKey application.ShardKey) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_ADDS} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphVertexRemovesKey(shardKey application.ShardKey) []byte { + key := []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphHyperedgeAddsKey(shardKey application.ShardKey) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func hypergraphHyperedgeRemovesKey(shardKey application.ShardKey) []byte { + key := []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES} + key = append(key, shardKey.L1[:]...) + key = append(key, shardKey.L2[:]...) + return key +} + +func shardKeyFromKey(key []byte) application.ShardKey { + return application.ShardKey{ + L1: [3]byte(key[2:5]), + L2: [32]byte(key[5:]), + } +} + +func (p *PebbleHypergraphStore) NewTransaction(indexed bool) ( + Transaction, + error, +) { + return p.db.NewBatch(indexed), nil +} + +func (p *PebbleHypergraphStore) LoadHypergraph() ( + *application.Hypergraph, + error, +) { + hg := application.NewHypergraph() + vertexAddsIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, VERTEX_ADDS}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer vertexAddsIter.Close() + for vertexAddsIter.First(); vertexAddsIter.Valid(); vertexAddsIter.Next() { + shardKey := make([]byte, len(vertexAddsIter.Key())) + copy(shardKey, vertexAddsIter.Key()) + + err := hg.ImportFromBytes( + application.VertexAtomType, + application.AddsPhaseType, + shardKeyFromKey(shardKey), + vertexAddsIter.Value(), + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + vertexRemovesIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES}, + []byte{HYPERGRAPH_SHARD, VERTEX_REMOVES + 1}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer vertexRemovesIter.Close() + for vertexRemovesIter.First(); vertexRemovesIter.Valid(); vertexRemovesIter.Next() { + shardKey := make([]byte, len(vertexRemovesIter.Key())) + copy(shardKey, vertexRemovesIter.Key()) + + err := hg.ImportFromBytes( + application.VertexAtomType, + application.RemovesPhaseType, + shardKeyFromKey(shardKey), + vertexRemovesIter.Value(), + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + hyperedgeAddsIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, HYPEREDGE_ADDS}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer hyperedgeAddsIter.Close() + for hyperedgeAddsIter.First(); hyperedgeAddsIter.Valid(); hyperedgeAddsIter.Next() { + shardKey := make([]byte, len(hyperedgeAddsIter.Key())) + copy(shardKey, hyperedgeAddsIter.Key()) + + err := hg.ImportFromBytes( + application.HyperedgeAtomType, + application.AddsPhaseType, + shardKeyFromKey(shardKey), + hyperedgeAddsIter.Value(), + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + hyperedgeRemovesIter, err := p.db.NewIter( + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES}, + []byte{HYPERGRAPH_SHARD, HYPEREDGE_REMOVES + 1}, + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + defer hyperedgeRemovesIter.Close() + for hyperedgeRemovesIter.First(); hyperedgeRemovesIter.Valid(); hyperedgeRemovesIter.Next() { + shardKey := make([]byte, len(hyperedgeRemovesIter.Key())) + copy(shardKey, hyperedgeRemovesIter.Key()) + + err := hg.ImportFromBytes( + application.HyperedgeAtomType, + application.RemovesPhaseType, + shardKeyFromKey(shardKey), + hyperedgeRemovesIter.Value(), + ) + if err != nil { + return nil, errors.Wrap(err, "load hypergraph") + } + } + + return hg, nil +} + +func (p *PebbleHypergraphStore) SaveHypergraph( + txn Transaction, + hg *application.Hypergraph, +) error { + for shardKey, vertexAdds := range hg.GetVertexAdds() { + if vertexAdds.IsDirty() { + err := txn.Set(hypergraphVertexAddsKey(shardKey), vertexAdds.ToBytes()) + if err != nil { + return errors.Wrap(err, "save hypergraph") + } + } + } + + for shardKey, vertexRemoves := range hg.GetVertexRemoves() { + if vertexRemoves.IsDirty() { + err := txn.Set( + hypergraphVertexRemovesKey(shardKey), + vertexRemoves.ToBytes(), + ) + if err != nil { + return errors.Wrap(err, "save hypergraph") + } + } + } + + for shardKey, hyperedgeAdds := range hg.GetHyperedgeAdds() { + if hyperedgeAdds.IsDirty() { + err := txn.Set( + hypergraphHyperedgeAddsKey(shardKey), + hyperedgeAdds.ToBytes(), + ) + if err != nil { + return errors.Wrap(err, "save hypergraph") + } + } + } + + for shardKey, hyperedgeRemoves := range hg.GetHyperedgeRemoves() { + if hyperedgeRemoves.IsDirty() { + err := txn.Set( + hypergraphHyperedgeRemovesKey(shardKey), + hyperedgeRemoves.ToBytes(), + ) + if err != nil { + return errors.Wrap(err, "save hypergraph") + } + } + } + + return nil +} diff --git a/node/test.sh b/node/test.sh index 9c88bbe..c221db7 100755 --- a/node/test.sh +++ b/node/test.sh @@ -11,6 +11,6 @@ BINARIES_DIR="$ROOT_DIR/target/release" # Link the native VDF and execute tests pushd "$NODE_DIR" > /dev/null - CGO_LDFLAGS="-L$BINARIES_DIR -L/opt/homebrew/Cellar/mpfr/4.2.1/lib -I/opt/homebrew/Cellar/mpfr/4.2.1/include -L/opt/homebrew/Cellar/gmp/6.3.0/lib -I/opt/homebrew/Cellar/gmp/6.3.0/include -L/opt/homebrew/Cellar/flint/3.1.3-p1/lib -I/opt/homebrew/Cellar/flint/3.1.3-p1/include -lbls48581 -lstdc++ -lvdf -ldl -lm -lflint -lgmp -lmpfr" \ + CGO_LDFLAGS="-L$BINARIES_DIR -lbls48581 -lverenc -lvdf -ldl -lm -lflint -lgmp -lmpfr" \ CGO_ENABLED=1 \ go test "$@" diff --git a/verenc/.gitignore b/verenc/.gitignore new file mode 100644 index 0000000..86d4c2d --- /dev/null +++ b/verenc/.gitignore @@ -0,0 +1 @@ +generated diff --git a/verenc/README.md b/verenc/README.md new file mode 100644 index 0000000..19c38dd --- /dev/null +++ b/verenc/README.md @@ -0,0 +1,9 @@ +# VerEnc + +Wrapper for the Rust implementation of Verifiable Encryption (VerEnc) in [crates/verenc](../crates/verenc). + +## Generate Go bindings + +```sh +go generate +``` diff --git a/verenc/generate.sh b/verenc/generate.sh new file mode 100755 index 0000000..553925f --- /dev/null +++ b/verenc/generate.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euxo pipefail + +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" + +RUST_VERENC_PACKAGE="$ROOT_DIR/crates/verenc" +BINDINGS_DIR="$ROOT_DIR/verenc" + +# Build the Rust VerEnc package in release mode +cargo build -p verenc --release + +# Generate Go bindings +pushd "$RUST_VERENC_PACKAGE" > /dev/null +uniffi-bindgen-go src/lib.udl -o "$BINDINGS_DIR"/generated diff --git a/verenc/go.mod b/verenc/go.mod new file mode 100644 index 0000000..bd1a6c9 --- /dev/null +++ b/verenc/go.mod @@ -0,0 +1,8 @@ +module source.quilibrium.com/quilibrium/monorepo/verenc + +go 1.20 + +require ( + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/sys v0.21.0 // indirect +) diff --git a/verenc/go.sum b/verenc/go.sum new file mode 100644 index 0000000..3983382 --- /dev/null +++ b/verenc/go.sum @@ -0,0 +1,4 @@ +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/verenc/test.sh b/verenc/test.sh new file mode 100755 index 0000000..a29568b --- /dev/null +++ b/verenc/test.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euxo pipefail + +# Run tests for the verenc package. Takes care of linking the native VerEnc library. +# Assumes that the VerEnc library has been built by running the generate.sh script in the same directory. + +ROOT_DIR="${ROOT_DIR:-$( cd "$(dirname "$(realpath "$( dirname "${BASH_SOURCE[0]}" )")")" >/dev/null 2>&1 && pwd )}" + +NODE_DIR="$ROOT_DIR/verenc" +BINARIES_DIR="$ROOT_DIR/target/release" + +# Link the native VerEnc library and execute tests +pushd "$NODE_DIR" > /dev/null + CGO_LDFLAGS="-L$BINARIES_DIR -lverenc -ldl -lm" \ + CGO_ENABLED=1 \ + go test "$@" diff --git a/verenc/verenc.go b/verenc/verenc.go new file mode 100644 index 0000000..104e365 --- /dev/null +++ b/verenc/verenc.go @@ -0,0 +1,35 @@ +package verenc + +import ( + generated "source.quilibrium.com/quilibrium/monorepo/verenc/generated/verenc" +) + +//go:generate ./generate.sh + +func NewVerencProof(data []byte) generated.VerencProofAndBlindingKey { + return generated.NewVerencProof(data) +} + +func NewVerencProofEncryptOnly(data []byte, encryptionKey []byte) generated.VerencProofAndBlindingKey { + return generated.NewVerencProofEncryptOnly(data, encryptionKey) +} + +func VerencVerify(proof generated.VerencProof) bool { + return generated.VerencVerify(proof) +} + +func VerencCompress(proof generated.VerencProof) generated.CompressedCiphertext { + return generated.VerencCompress(proof) +} + +func VerencRecover(recovery generated.VerencDecrypt) []byte { + return generated.VerencRecover(recovery) +} + +func ChunkDataForVerenc(data []byte) [][]byte { + return generated.ChunkDataForVerenc(data) +} + +func CombineChunkedData(chunks [][]byte) []byte { + return generated.CombineChunkedData(chunks) +} diff --git a/verenc/verenc_test.go b/verenc/verenc_test.go new file mode 100644 index 0000000..8257267 --- /dev/null +++ b/verenc/verenc_test.go @@ -0,0 +1,102 @@ +package verenc_test + +import ( + "bytes" + "crypto/rand" + "fmt" + "testing" + + "source.quilibrium.com/quilibrium/monorepo/verenc" + generated "source.quilibrium.com/quilibrium/monorepo/verenc/generated/verenc" +) + +func TestVerenc(t *testing.T) { + data := make([]byte, 56) + copy(data[1:6], []byte("hello")) + proof := verenc.NewVerencProof(data) + if !verenc.VerencVerify(generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }) { + t.FailNow() + } + compressed := verenc.VerencCompress(generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }) + recovered := verenc.VerencRecover(generated.VerencDecrypt{ + BlindingPubkey: proof.BlindingPubkey, + Statement: proof.Statement, + DecryptionKey: proof.DecryptionKey, + Ciphertexts: compressed, + }) + if !bytes.Equal(data, recovered) { + t.FailNow() + } +} + +func TestDataChunking(t *testing.T) { + data := make([]byte, 1300) + rand.Read(data) + chunks := verenc.ChunkDataForVerenc(data) + result := verenc.CombineChunkedData(chunks) + if !bytes.Equal(data, result[:1300]) { + t.FailNow() + } +} + +func TestVerencWithChunking(t *testing.T) { + data := make([]byte, 1300) + rand.Read(data) + chunks := verenc.ChunkDataForVerenc(data) + results := [][]byte{} + for i, chunk := range chunks { + proof := verenc.NewVerencProof(chunk) + if !verenc.VerencVerify(generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }) { + t.FailNow() + } + compressed := verenc.VerencCompress(generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }) + recovered := verenc.VerencRecover(generated.VerencDecrypt{ + BlindingPubkey: proof.BlindingPubkey, + Statement: proof.Statement, + DecryptionKey: proof.DecryptionKey, + Ciphertexts: compressed, + }) + if !bytes.Equal(chunk, recovered) { + fmt.Printf("recovered did not equal chunk %d: %x, %x\n", i, recovered, chunk) + t.FailNow() + } + results = append(results, recovered) + } + result := verenc.CombineChunkedData(results) + if !bytes.Equal(data, result[:1300]) { + fmt.Printf("result did not equal original data, %x, %x\n", result[:1300], data) + t.FailNow() + } +} From 5dab58a06bc8fb8bc39c53ea52cc1fad55c724ab Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 6 Feb 2025 06:37:39 -0600 Subject: [PATCH 08/13] add build step for macos --- Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Taskfile.yaml b/Taskfile.yaml index 27a2604..4e5a146 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -36,6 +36,7 @@ tasks: cmds: - vdf/generate.sh - bls48581/generate.sh + - verenc/generate.sh - node/build.sh -o build/arm64_macos/node build_qclient_arm64_macos: From 6deaee1439b50b2160a8eb032d9b84e4a65fe9a0 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 6 Feb 2025 06:49:24 -0600 Subject: [PATCH 09/13] fix client build imports --- client/build.sh | 4 ++-- client/go.mod | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/client/build.sh b/client/build.sh index 0019144..1d2757f 100755 --- a/client/build.sh +++ b/client/build.sh @@ -19,14 +19,14 @@ case "$os_type" in # Check if the architecture is ARM if [[ "$(uname -m)" == "arm64" ]]; then # MacOS ld doesn't support -Bstatic and -Bdynamic, so it's important that there is only a static version of the library - go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -L/opt/homebrew/Cellar/mpfr/4.2.1/lib -I/opt/homebrew/Cellar/mpfr/4.2.1/include -L/opt/homebrew/Cellar/gmp/6.3.0/lib -I/opt/homebrew/Cellar/gmp/6.3.0/include -L/opt/homebrew/Cellar/flint/3.1.3-p1/lib -I/opt/homebrew/Cellar/flint/3.1.3-p1/include -lbls48581 -lstdc++ -lvdf -ldl -lm -lflint -lgmp -lmpfr'" "$@" + go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -lbls48581 -lstdc++ -lvdf -lverenc -ldl -lm -lflint -lgmp -lmpfr'" "$@" else echo "Unsupported platform" exit 1 fi ;; "Linux") - go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -Wl,-Bstatic -lvdf -lbls48581 -Wl,-Bdynamic -lstdc++ -ldl -lm -lflint -lgmp -lmpfr'" "$@" + go build -ldflags "-linkmode 'external' -extldflags '-L$BINARIES_DIR -Wl,-Bstatic -lvdf -lbls48581 -lverenc -Wl,-Bdynamic -lstdc++ -ldl -lm -lflint -lgmp -lmpfr'" "$@" ;; *) echo "Unsupported platform" diff --git a/client/go.mod b/client/go.mod index 1ae2420..2e042d9 100644 --- a/client/go.mod +++ b/client/go.mod @@ -10,6 +10,8 @@ replace source.quilibrium.com/quilibrium/monorepo/bls48581 => ../bls48581 replace source.quilibrium.com/quilibrium/monorepo/vdf => ../vdf +replace source.quilibrium.com/quilibrium/monorepo/verenc => ../verenc + replace github.com/multiformats/go-multiaddr => ../go-multiaddr replace github.com/multiformats/go-multiaddr-dns => ../go-multiaddr-dns @@ -176,6 +178,7 @@ require ( source.quilibrium.com/quilibrium/monorepo/bls48581 v0.0.0-00010101000000-000000000000 // indirect source.quilibrium.com/quilibrium/monorepo/nekryptology v0.0.0-00010101000000-000000000000 // indirect source.quilibrium.com/quilibrium/monorepo/vdf v0.0.0-00010101000000-000000000000 // indirect + source.quilibrium.com/quilibrium/monorepo/verenc v0.0.0-00010101000000-000000000000 // indirect ) require ( From d61b78567b587a2c7cedbf45efd7e3686db9bd1e Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 6 Feb 2025 15:07:22 -0600 Subject: [PATCH 10/13] catch nil case --- .../intrinsics/token/token_execution_engine.go | 15 ++++++++++++--- node/hypergraph/application/hypergraph.go | 8 ++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index c68f8c4..6968399 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -236,6 +236,7 @@ func NewTokenExecutionEngine( keyManager: keyManager, clockStore: clockStore, coinStore: coinStore, + hypergraphStore: hypergraphStore, keyStore: keyStore, pubSub: pubSub, inclusionProver: inclusionProver, @@ -452,6 +453,10 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { for iter.First(); iter.Valid(); iter.Next() { key := make([]byte, len(iter.Key()[2:])) copy(key, iter.Key()[2:]) + e.logger.Debug( + "encrypting coin", + zap.String("address", hex.EncodeToString(key)), + ) data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) compressed := []hypergraph.Encrypted{} for _, d := range data { @@ -476,6 +481,10 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { for iter.First(); iter.Valid(); iter.Next() { key := make([]byte, len(iter.Key()[2:])) copy(key, iter.Key()[2:]) + e.logger.Debug( + "encrypting pre-coin proof", + zap.String("address", hex.EncodeToString(key)), + ) data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) compressed := []hypergraph.Encrypted{} for _, d := range data { @@ -492,14 +501,14 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { } } iter.Close() - e.logger.Info("saving rebuilt state tree") + e.logger.Info("saving rebuilt hypergraph") txn, err := e.clockStore.NewTransaction(false) if err != nil { panic(err) } - e.logger.Info("committing state tree") + e.logger.Info("committing hypergraph") roots := e.hypergraph.Commit() @@ -1179,7 +1188,7 @@ func (e *TokenExecutionEngine) ProcessFrame( } } - e.logger.Info("committing state tree") + e.logger.Info("committing hypergraph") roots := hg.Commit() diff --git a/node/hypergraph/application/hypergraph.go b/node/hypergraph/application/hypergraph.go index de5aff3..7565d3b 100644 --- a/node/hypergraph/application/hypergraph.go +++ b/node/hypergraph/application/hypergraph.go @@ -373,7 +373,11 @@ func (set *IdSet) Add(atom Atom) error { } func (set *IdSet) GetSize() *big.Int { - return set.tree.GetSize() + size := set.tree.GetSize() + if size == nil { + size = big.NewInt(0) + } + return size } func (set *IdSet) Delete(atom Atom) bool { @@ -407,7 +411,7 @@ type Hypergraph struct { func NewHypergraph() *Hypergraph { return &Hypergraph{ - size: new(big.Int), + size: big.NewInt(0), vertexAdds: make(map[ShardKey]*IdSet), vertexRemoves: make(map[ShardKey]*IdSet), hyperedgeAdds: make(map[ShardKey]*IdSet), From e5954a90a1582b8b94799a7fb1a430ec721ac4e3 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Thu, 6 Feb 2025 17:01:12 -0600 Subject: [PATCH 11/13] add smart caching for recurring payloads in verenc --- node/crypto/verifiable_encryption.go | 52 +++++++++++++++++++ .../token/token_execution_engine.go | 46 +++++++++++----- .../intrinsics/token/token_genesis.go | 14 +++-- 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/node/crypto/verifiable_encryption.go b/node/crypto/verifiable_encryption.go index c4dafce..983a1ef 100644 --- a/node/crypto/verifiable_encryption.go +++ b/node/crypto/verifiable_encryption.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "sync" + lru "github.com/hashicorp/golang-lru/v2" "source.quilibrium.com/quilibrium/monorepo/verenc" generated "source.quilibrium.com/quilibrium/monorepo/verenc/generated/verenc" ) @@ -124,6 +125,11 @@ func (p MPCitHVerEncProof) Verify() bool { return verenc.VerencVerify(p.VerencProof) } +type InlineEnc struct { + iv []byte + ciphertext []byte +} + func MPCitHVerEncFromBytes(data []byte) MPCitHVerEnc { ciphertext := generated.CompressedCiphertext{} for i := 0; i < 3; i++ { @@ -165,11 +171,18 @@ func (e MPCitHVerEnc) Verify(proof []byte) bool { type MPCitHVerifiableEncryptor struct { parallelism int + lruCache *lru.Cache[string, VerEnc] } func NewMPCitHVerifiableEncryptor(parallelism int) *MPCitHVerifiableEncryptor { + cache, err := lru.New[string, VerEnc](10000) + if err != nil { + panic(err) + } + return &MPCitHVerifiableEncryptor{ parallelism: parallelism, + lruCache: cache, } } @@ -205,6 +218,45 @@ func (v *MPCitHVerifiableEncryptor) Encrypt( return results } +func (v *MPCitHVerifiableEncryptor) EncryptAndCompress( + data []byte, + publicKey []byte, +) []VerEnc { + chunks := verenc.ChunkDataForVerenc(data) + results := make([]VerEnc, len(chunks)) + var wg sync.WaitGroup + throttle := make(chan struct{}, v.parallelism) + for i, chunk := range chunks { + throttle <- struct{}{} + wg.Add(1) + go func(chunk []byte, i int) { + defer func() { <-throttle }() + defer wg.Done() + existing, ok := v.lruCache.Get(string(publicKey) + string(chunk)) + if ok { + results[i] = existing + } else { + proof := verenc.NewVerencProofEncryptOnly(chunk, publicKey) + result := MPCitHVerEncProof{ + generated.VerencProof{ + BlindingPubkey: proof.BlindingPubkey, + EncryptionKey: proof.EncryptionKey, + Statement: proof.Statement, + Challenge: proof.Challenge, + Polycom: proof.Polycom, + Ctexts: proof.Ctexts, + SharesRands: proof.SharesRands, + }, + } + results[i] = result.Compress() + v.lruCache.Add(string(publicKey)+string(chunk), results[i]) + } + }(chunk, i) + } + wg.Wait() + return results +} + func (v *MPCitHVerifiableEncryptor) Decrypt( encrypted []VerEnc, decyptionKey []byte, diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 6968399..3dee0d1 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -457,11 +457,18 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { "encrypting coin", zap.String("address", hex.EncodeToString(key)), ) - data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) + data := e.mpcithVerEnc.EncryptAndCompress( + iter.Value(), + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range data { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } + e.logger.Debug( + "encrypted coin", + zap.String("address", hex.EncodeToString(key)), + ) if err := e.hypergraph.AddVertex( hypergraph.NewVertex( [32]byte(application.TOKEN_ADDRESS), @@ -485,10 +492,13 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { "encrypting pre-coin proof", zap.String("address", hex.EncodeToString(key)), ) - data := e.mpcithVerEnc.Encrypt(iter.Value(), config.GetGenesis().Beacon) + data := e.mpcithVerEnc.EncryptAndCompress( + iter.Value(), + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range data { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := e.hypergraph.AddVertex( hypergraph.NewVertex( @@ -726,10 +736,13 @@ func (e *TokenExecutionEngine) ProcessFrame( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, coinBytes...) - proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := e.mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.AddVertex( hypergraph.NewVertex( @@ -764,10 +777,13 @@ func (e *TokenExecutionEngine) ProcessFrame( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, coinBytes...) - proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := e.mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.RemoveVertex( hypergraph.NewVertex( @@ -803,10 +819,13 @@ func (e *TokenExecutionEngine) ProcessFrame( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, proofBytes...) - proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := e.mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.AddVertex( hypergraph.NewVertex( @@ -863,10 +882,13 @@ func (e *TokenExecutionEngine) ProcessFrame( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, proofBytes...) - proofs := e.mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := e.mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.RemoveVertex( hypergraph.NewVertex( diff --git a/node/execution/intrinsics/token/token_genesis.go b/node/execution/intrinsics/token/token_genesis.go index e16aa4e..9859716 100644 --- a/node/execution/intrinsics/token/token_genesis.go +++ b/node/execution/intrinsics/token/token_genesis.go @@ -880,10 +880,13 @@ func CreateGenesisState( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, coinBytes...) - proofs := mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.AddVertex( hypergraph.NewVertex( @@ -1048,10 +1051,13 @@ func CreateGenesisState( data := []byte{} data = binary.BigEndian.AppendUint64(data, 0) data = append(data, coinBytes...) - proofs := mpcithVerEnc.Encrypt(data, config.GetGenesis().Beacon) + proofs := mpcithVerEnc.EncryptAndCompress( + data, + config.GetGenesis().Beacon, + ) compressed := []hypergraph.Encrypted{} for _, d := range proofs { - compressed = append(compressed, d.Compress()) + compressed = append(compressed, d) } if err := hg.AddVertex( hypergraph.NewVertex( From b59e5b1565758c68184876e311a2fc8ce0545a5b Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Fri, 7 Feb 2025 00:05:05 -0600 Subject: [PATCH 12/13] don't rebuild hypergraph if unnecessary --- node/execution/intrinsics/token/token_execution_engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index 3dee0d1..e01dea8 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -362,8 +362,8 @@ func NewTokenExecutionEngine( e.proverPublicKey = publicKeyBytes e.provingKeyAddress = provingKeyAddress - frame, _, err := e.clockStore.GetLatestDataClockFrame(e.intrinsicFilter) - if err != nil || frame.FrameNumber < 186405 { + _, _, err = e.clockStore.GetLatestDataClockFrame(e.intrinsicFilter) + if err != nil { e.rebuildHypergraph() } else { e.hypergraph, err = e.hypergraphStore.LoadHypergraph() From 7afe704a2f245a2862c9f76f953814f50d61eb95 Mon Sep 17 00:00:00 2001 From: Cassandra Heart Date: Fri, 7 Feb 2025 16:31:43 -0600 Subject: [PATCH 13/13] if testnet don't run migration, further parallelization --- .../token/token_execution_engine.go | 262 ++++++++---------- .../intrinsics/token/token_genesis.go | 22 +- 2 files changed, 134 insertions(+), 150 deletions(-) diff --git a/node/execution/intrinsics/token/token_execution_engine.go b/node/execution/intrinsics/token/token_execution_engine.go index e01dea8..6d40a96 100644 --- a/node/execution/intrinsics/token/token_execution_engine.go +++ b/node/execution/intrinsics/token/token_execution_engine.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto" - "encoding/binary" "encoding/hex" "fmt" "math/big" @@ -171,27 +170,29 @@ func NewTokenExecutionEngine( } else if err != nil { panic(err) } else { - err := coinStore.Migrate( - intrinsicFilter, - config.GetGenesis().GenesisSeedHex, - ) - if err != nil { - panic(err) - } - _, err = clockStore.GetEarliestDataClockFrame(intrinsicFilter) - if err != nil && errors.Is(err, store.ErrNotFound) { - origin, inclusionProof, proverKeys, peerSeniority = CreateGenesisState( - logger, - cfg.Engine, - nil, - inclusionProver, - clockStore, - coinStore, - hypergraphStore, - hypergraph, - mpcithVerEnc, - uint(cfg.P2P.Network), + if pubSub.GetNetwork() == 0 { + err := coinStore.Migrate( + intrinsicFilter, + config.GetGenesis().GenesisSeedHex, ) + if err != nil { + panic(err) + } + _, err = clockStore.GetEarliestDataClockFrame(intrinsicFilter) + if err != nil && errors.Is(err, store.ErrNotFound) { + origin, inclusionProof, proverKeys, peerSeniority = CreateGenesisState( + logger, + cfg.Engine, + nil, + inclusionProver, + clockStore, + coinStore, + hypergraphStore, + hypergraph, + mpcithVerEnc, + uint(cfg.P2P.Network), + ) + } } } @@ -374,7 +375,7 @@ func NewTokenExecutionEngine( ) } - if e.hypergraph == nil { + if e.hypergraph == nil || len(e.hypergraph.GetVertexAdds()) == 0 { e.rebuildHypergraph() } } @@ -443,6 +444,50 @@ func NewTokenExecutionEngine( var _ execution.ExecutionEngine = (*TokenExecutionEngine)(nil) +func (e *TokenExecutionEngine) addBatchToHypergraph(batchKey [][]byte, batchValue [][]byte) { + var wg sync.WaitGroup + throttle := make(chan struct{}, runtime.NumCPU()) + batchCompressed := make([][]hypergraph.Encrypted, len(batchKey)) + for i, chunk := range batchValue { + throttle <- struct{}{} + wg.Add(1) + go func(chunk []byte, i int) { + defer func() { <-throttle }() + defer wg.Done() + e.logger.Debug( + "encrypting coin", + zap.String("address", hex.EncodeToString(batchKey[i])), + ) + data := e.mpcithVerEnc.EncryptAndCompress( + chunk, + config.GetGenesis().Beacon, + ) + compressed := []hypergraph.Encrypted{} + for _, d := range data { + compressed = append(compressed, d) + } + e.logger.Debug( + "encrypted coin", + zap.String("address", hex.EncodeToString(batchKey[i])), + ) + batchCompressed[i] = compressed + }(chunk, i) + } + wg.Wait() + + for i := range batchKey { + if err := e.hypergraph.AddVertex( + hypergraph.NewVertex( + [32]byte(application.TOKEN_ADDRESS), + [32]byte(batchKey[i]), + batchCompressed[i], + ), + ); err != nil { + panic(err) + } + } +} + func (e *TokenExecutionEngine) rebuildHypergraph() { e.logger.Info("rebuilding hypergraph") e.hypergraph = hypergraph.NewHypergraph() @@ -450,68 +495,40 @@ func (e *TokenExecutionEngine) rebuildHypergraph() { if err != nil { panic(err) } + var batchKey, batchValue [][]byte for iter.First(); iter.Valid(); iter.Next() { key := make([]byte, len(iter.Key()[2:])) copy(key, iter.Key()[2:]) - e.logger.Debug( - "encrypting coin", - zap.String("address", hex.EncodeToString(key)), - ) - data := e.mpcithVerEnc.EncryptAndCompress( - iter.Value(), - config.GetGenesis().Beacon, - ) - compressed := []hypergraph.Encrypted{} - for _, d := range data { - compressed = append(compressed, d) - } - e.logger.Debug( - "encrypted coin", - zap.String("address", hex.EncodeToString(key)), - ) - if err := e.hypergraph.AddVertex( - hypergraph.NewVertex( - [32]byte(application.TOKEN_ADDRESS), - [32]byte(key), - compressed, - ), - ); err != nil { + batchKey = append(batchKey, key) + + coin := &protobufs.Coin{} + err := proto.Unmarshal(iter.Value()[8:], coin) + if err != nil { panic(err) } + + value := []byte{} + value = append(value, iter.Value()[:8]...) + value = append(value, coin.Amount...) + // implicit + value = append(value, 0x00) + value = append(value, coin.Owner.GetImplicitAccount().GetAddress()...) + // domain len + value = append(value, 0x00) + value = append(value, coin.Intersection...) + batchValue = append(batchValue, value) + + if len(batchKey) == runtime.NumCPU() { + e.addBatchToHypergraph(batchKey, batchValue) + batchKey = [][]byte{} + batchValue = [][]byte{} + } } iter.Close() - iter, err = e.coinStore.RangePreCoinProofs() - if err != nil { - panic(err) + if len(batchKey) != 0 { + e.addBatchToHypergraph(batchKey, batchValue) } - for iter.First(); iter.Valid(); iter.Next() { - key := make([]byte, len(iter.Key()[2:])) - copy(key, iter.Key()[2:]) - e.logger.Debug( - "encrypting pre-coin proof", - zap.String("address", hex.EncodeToString(key)), - ) - data := e.mpcithVerEnc.EncryptAndCompress( - iter.Value(), - config.GetGenesis().Beacon, - ) - compressed := []hypergraph.Encrypted{} - for _, d := range data { - compressed = append(compressed, d) - } - if err := e.hypergraph.AddVertex( - hypergraph.NewVertex( - [32]byte(application.TOKEN_ADDRESS), - [32]byte(key), - compressed, - ), - ); err != nil { - panic(err) - } - } - iter.Close() - e.logger.Info("saving rebuilt hypergraph") txn, err := e.clockStore.NewTransaction(false) if err != nil { @@ -728,16 +745,22 @@ func (e *TokenExecutionEngine) ProcessFrame( txn.Abort() return nil, errors.Wrap(err, "process frame") } - coinBytes, err := proto.Marshal(o.Coin) - if err != nil { - panic(err) - } - data := []byte{} - data = binary.BigEndian.AppendUint64(data, 0) - data = append(data, coinBytes...) + value := []byte{} + value = append(value, make([]byte, 8)...) + value = append(value, o.Coin.Amount...) + // implicit + value = append(value, 0x00) + value = append( + value, + o.Coin.Owner.GetImplicitAccount().GetAddress()..., + ) + // domain len + value = append(value, 0x00) + value = append(value, o.Coin.Intersection...) + proofs := e.mpcithVerEnc.EncryptAndCompress( - data, + value, config.GetGenesis().Beacon, ) compressed := []hypergraph.Encrypted{} @@ -769,16 +792,22 @@ func (e *TokenExecutionEngine) ProcessFrame( txn.Abort() return nil, errors.Wrap(err, "process frame") } - coinBytes, err := proto.Marshal(coin) - if err != nil { - panic(err) - } - data := []byte{} - data = binary.BigEndian.AppendUint64(data, 0) - data = append(data, coinBytes...) + value := []byte{} + value = append(value, make([]byte, 8)...) + value = append(value, coin.Amount...) + // implicit + value = append(value, 0x00) + value = append( + value, + coin.Owner.GetImplicitAccount().GetAddress()..., + ) + // domain len + value = append(value, 0x00) + value = append(value, coin.Intersection...) + proofs := e.mpcithVerEnc.EncryptAndCompress( - data, + value, config.GetGenesis().Beacon, ) compressed := []hypergraph.Encrypted{} @@ -811,32 +840,7 @@ func (e *TokenExecutionEngine) ProcessFrame( txn.Abort() return nil, errors.Wrap(err, "process frame") } - proofBytes, err := proto.Marshal(o.Proof) - if err != nil { - panic(err) - } - data := []byte{} - data = binary.BigEndian.AppendUint64(data, 0) - data = append(data, proofBytes...) - proofs := e.mpcithVerEnc.EncryptAndCompress( - data, - config.GetGenesis().Beacon, - ) - compressed := []hypergraph.Encrypted{} - for _, d := range proofs { - compressed = append(compressed, d) - } - if err := hg.AddVertex( - hypergraph.NewVertex( - [32]byte(application.TOKEN_ADDRESS), - [32]byte(address), - compressed, - ), - ); err != nil { - txn.Abort() - panic(err) - } if len(o.Proof.Amount) == 32 && !bytes.Equal(o.Proof.Amount, make([]byte, 32)) && o.Proof.Commitment != nil { @@ -874,32 +878,6 @@ func (e *TokenExecutionEngine) ProcessFrame( txn.Abort() return nil, errors.Wrap(err, "process frame") } - proofBytes, err := proto.Marshal(o.DeletedProof) - if err != nil { - panic(err) - } - - data := []byte{} - data = binary.BigEndian.AppendUint64(data, 0) - data = append(data, proofBytes...) - proofs := e.mpcithVerEnc.EncryptAndCompress( - data, - config.GetGenesis().Beacon, - ) - compressed := []hypergraph.Encrypted{} - for _, d := range proofs { - compressed = append(compressed, d) - } - if err := hg.RemoveVertex( - hypergraph.NewVertex( - [32]byte(application.TOKEN_ADDRESS), - [32]byte(address), - compressed, - ), - ); err != nil { - txn.Abort() - panic(err) - } case *protobufs.TokenOutput_Announce: peerIds := []string{} for _, sig := range o.Announce.PublicKeySignaturesEd448 { diff --git a/node/execution/intrinsics/token/token_genesis.go b/node/execution/intrinsics/token/token_genesis.go index 9859716..e764070 100644 --- a/node/execution/intrinsics/token/token_genesis.go +++ b/node/execution/intrinsics/token/token_genesis.go @@ -872,16 +872,22 @@ func CreateGenesisState( if err != nil { panic(err) } - coinBytes, err := proto.Marshal(output.GetCoin()) - if err != nil { - panic(err) - } - data := []byte{} - data = binary.BigEndian.AppendUint64(data, 0) - data = append(data, coinBytes...) + value := []byte{} + value = append(value, make([]byte, 8)...) + value = append(value, output.GetCoin().Amount...) + // implicit + value = append(value, 0x00) + value = append( + value, + output.GetCoin().Owner.GetImplicitAccount().GetAddress()..., + ) + // domain len + value = append(value, 0x00) + value = append(value, output.GetCoin().Intersection...) + proofs := mpcithVerEnc.EncryptAndCompress( - data, + value, config.GetGenesis().Beacon, ) compressed := []hypergraph.Encrypted{}