coosld/tig-monorepo
1
0
Fork 0
mirror of https://github.com/tig-foundation/tig-monorepo.git synced 2026-02-21 10:27:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Optimise submit-benchmark payload size.

Browse Source
This commit is contained in:
FiveMovesAhead 2025-08-12 11:10:10 +01:00
parent e35b4af1c5
commit 34131bdf42
5 changed files with 104 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
swagger.yaml
View File

@ -61,7 +61,7 @@ paths:
* Returns all confirmed precommits, benchmarks, proofs, and frauds for the player where the benchmark was started within `120` blocks of the latest block.
* Fields `benchmark.solution_nonces`, `benchmark.discarded_solution_nonces`, `proof.merkle_proofs`, and `fraud.allegation` will always be `null`
* Fields `benchmark.solution_nonces`, `benchmark.discarded_solution_nonces`, `benchmark.non_solution_nonces`, `proof.merkle_proofs`, and `fraud.allegation` will always be `null`
* To retrieve that data, use /get-benchmark-data endpoint
parameters:
@ -90,8 +90,8 @@ paths:
summary: Get all data for a benchmark
description: |-
# Notes
* Will include data for fields `benchmark.solution_nonces`, `benchmark.discarded_solution_nonces`, `proof.merkle_proofs`, and `fraud.allegation`.
* Will include data for fields `benchmark.solution_nonces`, `benchmark.discarded_solution_nonces`, `benchmark.non_solution_nonces`, `proof.merkle_proofs`, and `fraud.allegation`.
parameters:
- name: benchmark_id
in: query
@ -689,11 +689,19 @@ components:
items:
type: integer
format: uint64
nullable: true
discarded_solution_nonces:
type: array
items:
type: integer
format: uint64
nullable: true
non_solution_nonces:
type: array
items:
type: integer
format: uint64
nullable: true
BenchmarkDetails:
type: object
properties:
@ -1110,6 +1118,7 @@ components:
$ref: '#/components/schemas/FraudState'
allegation:
type: string
nullable: true
FraudState:
type: object
properties:
@ -1322,6 +1331,7 @@ components:
type: array
items:
$ref: '#/components/schemas/MerkleProof'
nullable: true
ProofDetails:
type: object
properties:

2
tig-benchmarker/common/structs.py
View File

@ -81,6 +81,8 @@ class Benchmark(FromDict):
id: str
details: BenchmarkDetails
state: BenchmarkState
non_solution_nonces: Optional[Set[int]]
discarded_solution_nonces: Optional[Set[int]]
solution_nonces: Optional[Set[int]]
@dataclass

29
tig-benchmarker/master/master/submissions_manager.py
View File

@ -5,7 +5,7 @@ import logging
import os
from common.structs import *
from common.utils import *
from typing import Union, Set, List, Dict
from typing import Union, Set, List, Dict, Optional
from master.sql import get_db_conn
from master.client_manager import CONFIG
@ -20,8 +20,9 @@ class SubmitPrecommitRequest(FromDict):
class SubmitBenchmarkRequest(FromDict):
benchmark_id: str
merkle_root: MerkleHash
discarded_solution_nonces: Set[int]
solution_nonces: Set[int]
non_solution_nonces: Optional[List[int]]
discarded_solution_nonces: Optional[List[int]]
solution_nonces: Optional[List[int]]
@dataclass
class SubmitProofRequest(FromDict):
@ -116,10 +117,11 @@ class SubmissionsManager:
ORDER BY block_started
LIMIT 1
)
RETURNING benchmark_id
RETURNING benchmark_id, num_nonces
)
SELECT
B.benchmark_id,
SELECT
A.benchmark_id,
A.num_nonces,
B.merkle_root,
B.solution_nonces,
B.discarded_solution_nonces
@ -132,13 +134,28 @@ class SubmissionsManager:
if benchmark_to_submit:
benchmark_id = benchmark_to_submit["benchmark_id"]
num_nonces = benchmark_to_submit["num_nonces"]
merkle_root = benchmark_to_submit["merkle_root"]
non_solution_nonces = list(
set(range(num_nonces)) -
set(benchmark_to_submit["solution_nonces"]) -
set(benchmark_to_submit["discarded_solution_nonces"])
)
solution_nonces = benchmark_to_submit["solution_nonces"]
discarded_solution_nonces = benchmark_to_submit["discarded_solution_nonces"]
max_size = (num_nonces + 2) // 3
if len(solution_nonces) > max_size:
solution_nonces = None
elif len(discarded_solution_nonces) > max_size:
discarded_solution_nonces = None
else:
non_solution_nonces = None
self._post_thread("benchmark", SubmitBenchmarkRequest(
benchmark_id=benchmark_id,
merkle_root=merkle_root,
non_solution_nonces=non_solution_nonces,
solution_nonces=solution_nonces,
discarded_solution_nonces=discarded_solution_nonces,
))

5
tig-protocol/src/context.rs
View File

@ -19,8 +19,9 @@ pub trait Context {
&self,
benchmark_id: String,
details: BenchmarkDetails,
solution_nonces: HashSet<u64>,
discarded_solution_nonces: HashSet<u64>,
non_solution_nonces: Option<HashSet<u64>>,
solution_nonces: Option<HashSet<u64>>,
discarded_solution_nonces: Option<HashSet<u64>>,
) -> Result<()>;
async fn get_binary_details(&self, code_id: &String) -> Option<BinaryDetails>;
async fn add_binary_to_mempool(&self, code_id: String, details: BinaryDetails) -> Result<()>;

118
tig-protocol/src/contracts/benchmarks.rs
View File

@ -1,5 +1,6 @@
use crate::context::*;
use anyhow::{anyhow, Result};
use core::num;
use logging_timer::time;
use rand::{rngs::StdRng, seq::IteratorRandom, Rng, SeedableRng};
use std::collections::HashSet;
@ -114,8 +115,9 @@ pub async fn submit_benchmark<T: Context>(
player_id: String,
benchmark_id: String,
merkle_root: MerkleHash,
solution_nonces: HashSet<u64>,
discarded_solution_nonces: HashSet<u64>,
non_solution_nonces: Option<HashSet<u64>>,
solution_nonces: Option<HashSet<u64>>,
discarded_solution_nonces: Option<HashSet<u64>>,
seed: u64,
) -> Result<()> {
// check benchmark is not duplicate
@ -136,83 +138,89 @@ pub async fn submit_benchmark<T: Context>(
));
}
// check solution nonces is valid
// check at least 2 sets of nonces are provided
let precommit_details = ctx.get_precommit_details(&benchmark_id).await.unwrap();
let num_nonces = precommit_details.num_nonces as u64;
if !solution_nonces.iter().all(|n| *n < num_nonces) {
return Err(anyhow!("Invalid solution nonces"));
let max_set_size = ((num_nonces + 2) / 3) as usize;
let mut nonces_sets = vec![
&solution_nonces,
&discarded_solution_nonces,
&non_solution_nonces,
];
nonces_sets.sort_by_key(|x| x.is_none());
if nonces_sets[1].is_none() || nonces_sets[2].is_some() {
return Err(anyhow!("Exactly 2 sets of nonces must be provided"));
}
let set_a = nonces_sets[0].as_ref().unwrap();
let set_b = nonces_sets[1].as_ref().unwrap();
if !set_a.is_disjoint(set_b) {
return Err(anyhow!("Nonces sets must be disjoint.",));
}
if set_a.len() > max_set_size || set_b.len() > max_set_size {
return Err(anyhow!("The 2 smaller sets of nonces must be submitted"));
}
if !set_a.iter().all(|n| *n < num_nonces) || !set_b.iter().all(|n| *n < num_nonces) {
return Err(anyhow!("Invalid nonces"));
}
// random sample nonces
let config = ctx.get_config().await;
let mut rng = StdRng::seed_from_u64(seed);
let benchmark_config = &config.challenges[&settings.challenge_id].benchmarks;
let benchmark_config = &config.challenges[&settings.challenge_id]
.benchmarks
.max_samples;
let max_samples = benchmark_config.max_samples;
// sample nonces from solutions
let mut sampled_solution_nonces = HashSet::new();
if !solution_nonces.is_empty() {
for _ in 0..25 {
if sampled_solution_nonces.len() == max_samples {
break;
}
sampled_solution_nonces.insert(*solution_nonces.iter().choose(&mut rng).unwrap());
}
}
// sample nonces from discarded solutions
let mut sampled_discarded_solution_nonces = HashSet::new();
if !discarded_solution_nonces.is_empty() {
for _ in 0..25 {
if sampled_discarded_solution_nonces.len() == max_samples {
break;
}
sampled_discarded_solution_nonces
.insert(*discarded_solution_nonces.iter().choose(&mut rng).unwrap());
}
}
// sample nonces from non-solutions
let mut sampled_non_solution_nonces = HashSet::new();
let num_non_solution_nonces =
num_nonces - solution_nonces.len() as u64 - discarded_solution_nonces.len() as u64;
if num_non_solution_nonces > 0 {
if num_non_solution_nonces * 2 <= num_nonces {
let non_solution_nonces: HashSet<u64> = (0..num_nonces)
.filter(|n| !solution_nonces.contains(n) && !discarded_solution_nonces.contains(n))
.collect();
for _ in 0..25 {
if sampled_non_solution_nonces.len() == max_samples {
break;
let mut sampled_nonces = HashSet::new();
for set_x in [
&solution_nonces,
&discarded_solution_nonces,
&non_solution_nonces,
] {
let break_size = sampled_nonces.len() + max_samples;
if let Some(set_x) = set_x {
if !set_x.is_empty() {
for _ in 0..25 {
if sampled_nonces.len() == break_size {
break;
}
sampled_nonces.insert(*set_x.iter().choose(&mut rng).unwrap());
}
sampled_non_solution_nonces
.insert(*non_solution_nonces.iter().choose(&mut rng).unwrap());
}
} else {
// if there are more non-solutions than solutions, sample from all non-solutions
// this set is at least 1/3 of the total nonces
for _ in 0..25 {
if sampled_non_solution_nonces.len() == max_samples {
if sampled_nonces.len() == break_size {
break;
}
sampled_non_solution_nonces.insert(rng.gen_range(0..num_nonces));
let nonce = rng.gen_range(0..num_nonces);
if !set_a.contains(&nonce) && !set_b.contains(&nonce) {
sampled_nonces.insert(nonce);
}
}
}
}
let sampled_nonces: HashSet<u64> = sampled_solution_nonces
.into_iter()
.chain(sampled_non_solution_nonces.into_iter())
.chain(sampled_discarded_solution_nonces.into_iter())
.collect();
let num_solutions = if let Some(solution_nonces) = &solution_nonces {
solution_nonces.len()
} else {
num_nonces as usize - set_a.len() - set_b.len()
} as u32;
let num_discarded_solutions =
if let Some(discarded_solution_nonces) = &discarded_solution_nonces {
discarded_solution_nonces.len()
} else {
num_nonces as usize - set_a.len() - set_b.len()
} as u32;
ctx.add_benchmark_to_mempool(
benchmark_id,
BenchmarkDetails {
num_solutions: solution_nonces.len() as u32,
num_discarded_solutions: discarded_solution_nonces.len() as u32,
num_solutions,
num_discarded_solutions,
merkle_root,
sampled_nonces,
},
non_solution_nonces,
solution_nonces,
discarded_solution_nonces,
)