diff --git a/tig-benchmarker/common/structs.py b/tig-benchmarker/common/structs.py index 894f852d..9ae399f6 100644 --- a/tig-benchmarker/common/structs.py +++ b/tig-benchmarker/common/structs.py @@ -56,7 +56,7 @@ class PrecommitDetails(FromDict): rand_hash: str fee_paid: PreciseNumber hyperparameters: Optional[dict] - runtime_config: dict + fuel_budget: int @dataclass class PrecommitState(FromDict): diff --git a/tig-benchmarker/master/master/job_manager.py b/tig-benchmarker/master/master/job_manager.py index 8ff73800..99e45197 100644 --- a/tig-benchmarker/master/master/job_manager.py +++ b/tig-benchmarker/master/master/job_manager.py @@ -82,7 +82,7 @@ class JobManager: num_nonces, num_batches, rand_hash, - runtime_config, + fuel_budget, batch_size, challenge, algorithm, @@ -100,7 +100,7 @@ class JobManager: x.details.num_nonces, num_batches, x.details.rand_hash, - json.dumps(x.details.runtime_config), + x.details.fuel_budget, batch_size, c_name, a_name, diff --git a/tig-benchmarker/master/master/precommit_manager.py b/tig-benchmarker/master/master/precommit_manager.py index ffcfd14f..ffeaac3e 100644 --- a/tig-benchmarker/master/master/precommit_manager.py +++ b/tig-benchmarker/master/master/precommit_manager.py @@ -47,21 +47,24 @@ class PrecommitManager: logger.error(f"Invalid selected challenge_id '{c_id}'. Valid challenge_ids: {sorted(self.challenge_configs)}") return challenge_config = self.challenge_configs[c_id] - selected_track_ids = sorted(set(selection["selected_track_ids"]) & set(challenge_config["active_tracks"])) - if len(selected_track_ids) == 0: - selected_track_ids = sorted(challenge_config["active_tracks"]) - selection["selected_track_ids"] = selected_track_ids + for t_id in set(selection["track_settings"]) - set(challenge_config["active_tracks"]): + selection["track_settings"].pop(t_id) + for t_id in set(challenge_config["active_tracks"]) - set(selection["track_settings"]): + selection["track_settings"][t_id] = {} - if selection["num_bundles"] < challenge_config["min_num_bundles"]: - selection["num_bundles"] = challenge_config["min_num_bundles"] - - if ( - len(selection["runtime_config"]) > 1 or - selection["runtime_config"].get("max_fuel") is None or - selection["runtime_config"]["max_fuel"] < 0 or - selection["runtime_config"]["max_fuel"] > challenge_config["runtime_config_limits"]["max_fuel"] - ): - selection["runtime_config"] = {"max_fuel": challenge_config["runtime_config_limits"]["max_fuel"]} + for t_id in set(challenge_config["active_tracks"]): + for k in set(selection["track_settings"][t_id]) - {"num_bundles", "hyperparameters", "fuel_budget"}: + selection["track_settings"][t_id].pop(k) + if selection["track_settings"][t_id].get("num_bundles", 0) < challenge_config["min_num_bundles"]: + selection["track_settings"][t_id]["num_bundles"] = challenge_config["min_num_bundles"] + if ( + selection["track_settings"][t_id].get("fuel_budget") is None or + selection["track_settings"][t_id]["fuel_budget"] < 0 or + selection["track_settings"][t_id]["fuel_budget"] > challenge_config["max_fuel_budget"] + ): + selection["track_settings"][t_id]["fuel_budget"] = challenge_config["max_fuel_budget"] + if "hyperparameters" not in selection["track_settings"][t_id]: + selection["track_settings"][t_id]["hyperparameters"] = None self.num_precommits_submitted += 1 req = SubmitPrecommitRequest( @@ -70,14 +73,9 @@ class PrecommitManager: algorithm_id=a_id, player_id=CONFIG["player_id"], block_id=self.last_block_id, - track_id=random.choice(selection["selected_track_ids"]), + track_id="", ), - num_bundles=selection["num_bundles"], - hyperparameters=selection["hyperparameters"], - runtime_config={ - **challenge_config["runtime_config_limits"], - **selection["runtime_config"] - }, + track_settings=selection["track_settings"], ) logger.info(f"Created precommit (algorithm_id: {a_id}, track: {req.settings.track_id}, num_bundles: {req.num_bundles}, hyperparameters: {req.hyperparameters}, runtime_config: {req.runtime_config})") return req \ No newline at end of file diff --git a/tig-benchmarker/master/master/slave_manager.py b/tig-benchmarker/master/master/slave_manager.py index fccc4926..b73ae4cf 100644 --- a/tig-benchmarker/master/master/slave_manager.py +++ b/tig-benchmarker/master/master/slave_manager.py @@ -43,7 +43,7 @@ class SlaveManager: 'settings', B.settings, 'hyperparameters', B.hyperparameters, 'sampled_nonces', A.sampled_nonces, - 'runtime_config', B.runtime_config, + 'fuel_budget', B.fuel_budget, 'download_url', B.download_url, 'rand_hash', B.rand_hash, 'batch_size', B.batch_size, @@ -76,7 +76,7 @@ class SlaveManager: 'settings', B.settings, 'hyperparameters', B.hyperparameters, 'sampled_nonces', NULL, - 'runtime_config', B.runtime_config, + 'fuel_budget', B.fuel_budget, 'download_url', B.download_url, 'rand_hash', B.rand_hash, 'batch_size', B.batch_size, diff --git a/tig-benchmarker/master/master/submissions_manager.py b/tig-benchmarker/master/master/submissions_manager.py index 3d73d140..0d170526 100644 --- a/tig-benchmarker/master/master/submissions_manager.py +++ b/tig-benchmarker/master/master/submissions_manager.py @@ -12,11 +12,15 @@ from master.client_manager import CONFIG logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) @dataclass -class SubmitPrecommitRequest(FromDict): - settings: BenchmarkSettings +class TrackSettings(FromDict): num_bundles: int hyperparameters: Optional[dict] - runtime_config: dict + fuel_budget: int + +@dataclass +class SubmitPrecommitRequest(FromDict): + settings: BenchmarkSettings + track_settings: Dict[str, TrackSettings] @dataclass class SubmitBenchmarkRequest(FromDict): diff --git a/tig-benchmarker/postgres/init.sql b/tig-benchmarker/postgres/init.sql index 7b1b2b08..482d73b5 100644 --- a/tig-benchmarker/postgres/init.sql +++ b/tig-benchmarker/postgres/init.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS job ( hyperparameters JSONB, num_nonces INTEGER NOT NULL, rand_hash TEXT NOT NULL, - runtime_config JSONB NOT NULL, + fuel_budget BIGINT NOT NULL, batch_size INTEGER NOT NULL, num_batches INTEGER NOT NULL, challenge TEXT NOT NULL, @@ -115,57 +115,39 @@ SELECT ' "algo_selection": [ { "algorithm_id": "c001_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 }, { "algorithm_id": "c002_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 }, { "algorithm_id": "c003_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 }, { "algorithm_id": "c004_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 }, { "algorithm_id": "c005_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 }, { "algorithm_id": "c006_a001", - "num_bundles": 1, - "selected_track_ids": [], + "track_settings": {}, "weight": 1, - "batch_size": 8, - "hyperparameters": null, - "runtime_config": {} + "batch_size": 8 } ], "time_before_batch_retry": 60000, diff --git a/tig-benchmarker/slave/main.py b/tig-benchmarker/slave/main.py index 2c660fb1..8040d867 100644 --- a/tig-benchmarker/slave/main.py +++ b/tig-benchmarker/slave/main.py @@ -68,7 +68,7 @@ def run_tig_runtime(nonce, batch, so_path, ptx_path, results_dir): batch["rand_hash"], str(nonce), so_path, - "--fuel", str(batch["runtime_config"]["max_fuel"]), + "--fuel", str(batch["fuel_budget"]), "--output", output_dir, ] if batch["hyperparameters"] is not None: diff --git a/tig-protocol/src/contracts/benchmarks.rs b/tig-protocol/src/contracts/benchmarks.rs index d086a6e5..6758661c 100644 --- a/tig-protocol/src/contracts/benchmarks.rs +++ b/tig-protocol/src/contracts/benchmarks.rs @@ -1,19 +1,16 @@ use crate::context::*; use anyhow::{anyhow, Result}; use logging_timer::time; -use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; -use serde_json::{Map, Value}; -use std::collections::HashSet; +use rand::{prelude::IteratorRandom, rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; +use std::collections::{HashMap, HashSet}; use tig_structs::{config::*, core::*}; #[time] pub async fn submit_precommit( ctx: &T, player_id: String, - settings: BenchmarkSettings, - hyperparameters: Option>, - runtime_config: RuntimeConfig, - num_bundles: u64, + mut settings: BenchmarkSettings, + mut track_settings: HashMap, seed: u64, ) -> Result { if player_id != settings.player_id { @@ -53,14 +50,33 @@ pub async fn submit_precommit( // verify size let challenge_config = &config.challenges[&settings.challenge_id]; - if !challenge_config - .active_tracks - .contains_key(&settings.track_id) + if challenge_config.active_tracks.len() != track_settings.len() + || !track_settings + .keys() + .all(|k| challenge_config.active_tracks.contains_key(k)) { - return Err(anyhow!("Invalid track_id '{}'", settings.track_id)); + return Err(anyhow!( + "Must submit settings for all active tracks: {:?}", + challenge_config.active_tracks.keys().collect::>(), + )); } + + // randomly select a track + let mut rng = StdRng::seed_from_u64(seed); + settings.track_id = challenge_config + .active_tracks + .keys() + .choose(&mut rng) + .unwrap() + .clone(); let track_config = &challenge_config.active_tracks[&settings.track_id]; + let TrackSettings { + hyperparameters, + fuel_budget, + num_bundles, + } = track_settings.remove(&settings.track_id).unwrap(); + if num_bundles < challenge_config.min_num_bundles { return Err(anyhow!( "Invalid num_bundles '{}'. Must be at least {}", @@ -69,19 +85,11 @@ pub async fn submit_precommit( )); } - if runtime_config.max_memory > challenge_config.runtime_config_limits.max_memory { + if fuel_budget > challenge_config.max_fuel_budget { return Err(anyhow!( - "Invalid runtime_config.max_memory '{}'. Must be <= {}", - runtime_config.max_memory, - challenge_config.runtime_config_limits.max_memory - )); - } - - if runtime_config.max_fuel > challenge_config.runtime_config_limits.max_fuel { - return Err(anyhow!( - "Invalid runtime_config.max_fuel '{}'. Must be <= {}", - runtime_config.max_fuel, - challenge_config.runtime_config_limits.max_fuel + "Invalid fuel_budget '{}'. Must be <= {}", + fuel_budget, + challenge_config.max_fuel_budget )); } @@ -103,10 +111,10 @@ pub async fn submit_precommit( block_started: block_details.height, num_nonces: num_bundles * track_config.num_nonces_per_bundle, num_bundles, - rand_hash: hex::encode(StdRng::seed_from_u64(seed).gen::<[u8; 16]>()), + rand_hash: hex::encode(rng.r#gen::<[u8; 16]>()), fee_paid: submission_fee, hyperparameters, - runtime_config, + fuel_budget, }, ) .await?; diff --git a/tig-structs/src/config.rs b/tig-structs/src/config.rs index 08af1ec5..56983bf4 100644 --- a/tig-structs/src/config.rs +++ b/tig-structs/src/config.rs @@ -51,12 +51,6 @@ serializable_struct_with_getters! { token_locker_weight: u32, } } -serializable_struct_with_getters! { - RuntimeConfig { - max_memory: u64, - max_fuel: u64, - } -} serializable_struct_with_getters! { TopUpsConfig { topup_address: String, @@ -93,7 +87,7 @@ serializable_struct_with_getters! { per_nonce_fee: PreciseNumber, base_fee: PreciseNumber, active_tracks: HashMap, - runtime_config_limits: RuntimeConfig, + max_fuel_budget: u64, max_qualifiers_per_track: u64, legacy_multiplier_span: f32, min_num_bundles: u64, diff --git a/tig-structs/src/core.rs b/tig-structs/src/core.rs index e9dbac2e..bc8f3daa 100644 --- a/tig-structs/src/core.rs +++ b/tig-structs/src/core.rs @@ -1,5 +1,5 @@ use crate::{ - config::{ChallengeConfig, ProtocolConfig, RuntimeConfig}, + config::{ChallengeConfig, ProtocolConfig}, serializable_struct_with_getters, }; use serde::{Deserialize, Serialize}; @@ -397,6 +397,13 @@ serializable_struct_with_getters! { } // Precommit child structs +serializable_struct_with_getters! { + TrackSettings { + hyperparameters: Option>, + fuel_budget: u64, + num_bundles: u64, + } +} serializable_struct_with_getters! { PrecommitDetails { block_started: u32, @@ -404,7 +411,7 @@ serializable_struct_with_getters! { num_bundles: u64, rand_hash: String, fee_paid: PreciseNumber, - runtime_config: RuntimeConfig, + fuel_budget: u64, hyperparameters: Option>, } }