From a0995e01a81ed303a8beb14a2d296e3ba967ba4b Mon Sep 17 00:00:00 2001 From: FiveMovesAhead Date: Tue, 25 Nov 2025 16:51:50 +0000 Subject: [PATCH] Update benchmarker to work with tracks. --- swagger.yaml | 50 ++++++------------- tig-benchmarker/common/structs.py | 21 ++++---- tig-benchmarker/master/main.py | 6 +-- .../master/master/client_manager.py | 8 +-- tig-benchmarker/master/master/job_manager.py | 16 +++--- .../master/master/precommit_manager.py | 8 +-- tig-benchmarker/master/master/race_sampler.py | 38 -------------- .../master/master/slave_manager.py | 2 +- .../master/master/submissions_manager.py | 2 +- tig-benchmarker/postgres/init.sql | 28 +++++------ tig-benchmarker/tests/data.py | 6 +-- .../ui/src/app/pages/home/home.component.html | 12 ++--- .../ui/src/app/services/tig-apis.service.ts | 4 +- 13 files changed, 69 insertions(+), 132 deletions(-) delete mode 100644 tig-benchmarker/master/master/race_sampler.py diff --git a/swagger.yaml b/swagger.yaml index 83ea1267..5e4d53c7 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -172,15 +172,15 @@ paths: application/json: schema: $ref: '#/components/schemas/GetDelegatorsResponse' - /get-difficulty-data: + /get-tracks-data: get: tags: - GET - summary: Get latest difficulty data for a specific challenge id + summary: Get latest tracks data for a specific challenge id description: |- # Notes - * Returns aggregated num_nonces, num_solutions, difficulty data from benchmarks in the last 120 blocks. + * Returns aggregated algorithm_id, average_quality, num_bundles data from benchmarks in the last 120 blocks. * Query parameter `` must be the latest block. Use `/get-block` endpoint @@ -202,7 +202,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetDifficultyDataResponse' + $ref: '#/components/schemas/GetTracksDataResponse' /get-challenges: get: tags: @@ -651,7 +651,7 @@ components: type: string pattern: ^c[0-9]{3}_a[0-9]{3}$ example: c002_a001 - RaceId: + TrackId: type: string pattern: ^\w+=\w+(,\w+=\w+)*$ example: num_variables=100,clauses_to_variables_percent=426 @@ -736,8 +736,8 @@ components: $ref: '#/components/schemas/ChallengeId' algorithm_id: $ref: '#/components/schemas/AlgorithmId' - race_id: - $ref: '#/components/schemas/RaceId' + track_id: + $ref: '#/components/schemas/TrackId' BenchmarkState: type: object properties: @@ -1011,8 +1011,8 @@ components: properties: id: $ref: '#/components/schemas/ChallengeId' - details: - $ref: '#/components/schemas/ChallengeDetails' + config: + type: object state: $ref: '#/components/schemas/ChallengeState' block_data: @@ -1024,20 +1024,12 @@ components: type: integer format: uint64 qualifier_qualities: - description: "Map" + description: "Map" additionalProperties: type: array items: type: integer format: int32 - ChallengeDetails: - type: object - properties: - name: - type: string - type: - type: string - enum: [cpu, gpu] ChallengeId: type: string pattern: ^c[0-9]{3}$ @@ -1081,21 +1073,15 @@ components: block_confirmed: type: integer format: uint32 - Difficulty: - type: array - items: - type: integer - format: int32 - example: [40, 250] - DifficultyData: + TracksData: type: object properties: algorithm_id: $ref: '#/components/schemas/AlgorithmId' - num_nonces: + num_bundles: type: integer format: uint64 - average_solution_quality: + average_quality: type: integer format: int32 Fraud: @@ -1114,10 +1100,6 @@ components: block_confirmed: type: integer format: uint32 - Frontier: - type: array - items: - $ref: '#/components/schemas/Difficulty' MD5: type: string pattern: ^[a-f0-9]{32}$ @@ -1452,16 +1434,16 @@ components: format: double reward: $ref: '#/components/schemas/PreciseNumber' - GetDifficultyDataResponse: + GetTracksDataResponse: type: object properties: data: type: object - description: "Map" + description: "Map" additionalProperties: type: array items: - $ref: '#/components/schemas/DifficultyData' + $ref: '#/components/schemas/TracksData' GetOPoWResponse: type: object properties: diff --git a/tig-benchmarker/common/structs.py b/tig-benchmarker/common/structs.py index 77670343..702e050e 100644 --- a/tig-benchmarker/common/structs.py +++ b/tig-benchmarker/common/structs.py @@ -25,7 +25,7 @@ class CodeState(FromDict): @dataclass class CodeBlockData(FromDict): - num_qualifiers_by_player: Dict[str, int] + num_qualifiers_by_track_by_player: Dict[str, int] adoption: PreciseNumber merge_points: int reward: PreciseNumber @@ -43,7 +43,7 @@ class BenchmarkSettings(FromDict): block_id: str challenge_id: str algorithm_id: str - race_id: str + track_id: str def calc_seed(self, rand_hash: str, nonce: int) -> bytes: return u8s_from_str(f"{jsonify(self)}_{rand_hash}_{nonce}") @@ -52,6 +52,7 @@ class BenchmarkSettings(FromDict): class PrecommitDetails(FromDict): block_started: int num_nonces: int + num_bundles: int rand_hash: str fee_paid: PreciseNumber hyperparameters: Optional[dict] @@ -72,7 +73,7 @@ class Precommit(FromDict): class BenchmarkDetails(FromDict): stopped: bool merkle_root: Optional[MerkleHash] - average_solution_quality: Optional[int] + average_quality_by_bundle: Optional[List[int]] sampled_nonces: Optional[List[int]] @dataclass @@ -175,29 +176,25 @@ class Block(FromDict): config: dict data: Optional[BlockData] -@dataclass -class ChallengeDetails(FromDict): - name: str - @dataclass class ChallengeState(FromDict): round_active: int @dataclass class ChallengeBlockData(FromDict): - num_qualifiers: int + num_qualifiers_by_track: int qualifier_qualities: Dict[str, List[int]] @dataclass class Challenge(FromDict): id: str - details: ChallengeDetails + config: dict state: ChallengeState block_data: Optional[ChallengeBlockData] @dataclass class OPoWBlockData(FromDict): - num_qualifiers_by_challenge: Dict[str, int] + num_qualifiers_by_challenge_by_track: Dict[str, int] cutoff: int weighted_delegated_deposit: PreciseNumber weighted_self_deposit: PreciseNumber @@ -274,8 +271,8 @@ class TopUp(FromDict): @dataclass class DifficultyData(FromDict): - average_solution_quality: int - num_nonces: int + average_quality: int + num_bundles: int algorithm_id: str @dataclass diff --git a/tig-benchmarker/master/main.py b/tig-benchmarker/master/main.py index f64177ca..0d077ffd 100644 --- a/tig-benchmarker/master/main.py +++ b/tig-benchmarker/master/main.py @@ -2,7 +2,6 @@ import logging import os import time from master.data_fetcher import * -from master.race_sampler import * from master.job_manager import * from master.precommit_manager import * from master.slave_manager import * @@ -19,7 +18,6 @@ def main(): client_manager.start() data_fetcher = DataFetcher() - race_sampler = RaceSampler() job_manager = JobManager() precommit_manager = PrecommitManager() submissions_manager = SubmissionsManager() @@ -33,13 +31,11 @@ def main(): if data["block"].id != last_block_id: last_block_id = data["block"].id client_manager.on_new_block(**data) - race_sampler.on_new_block(**data) job_manager.on_new_block(**data) submissions_manager.on_new_block(**data) precommit_manager.on_new_block(**data) job_manager.run() - samples = race_sampler.run() - submit_precommit_req = precommit_manager.run(samples) + submit_precommit_req = precommit_manager.run() submissions_manager.run(submit_precommit_req) slave_manager.run() except Exception as e: diff --git a/tig-benchmarker/master/master/client_manager.py b/tig-benchmarker/master/master/client_manager.py index b984ffb6..864abcae 100644 --- a/tig-benchmarker/master/master/client_manager.py +++ b/tig-benchmarker/master/master/client_manager.py @@ -122,7 +122,7 @@ class ClientManager: B.batch_size, B.num_nonces - B.batch_size * C.batch_idx ), - 'average_solution_quality', D.average_solution_quality, + 'average_quality', D.average_quality, 'num_attempts', COALESCE(E.num_attempts, C.num_attempts), 'status', CASE WHEN B.stopped IS NOT NULL THEN 'STOPPED' @@ -167,9 +167,9 @@ class ClientManager: benchmark_id, JSONB_AGG(batch_data ORDER BY batch_idx) AS batches, ( - SUM((batch_data->>'num_nonces')::BIGINT * (batch_data->>'average_solution_quality')::BIGINT) / + SUM((batch_data->>'num_nonces')::BIGINT * (batch_data->>'average_quality')::BIGINT) / SUM((batch_data->>'num_nonces')::BIGINT) - )::INTEGER AS average_solution_quality + )::INTEGER AS average_quality FROM recent_batches GROUP BY benchmark_id ) @@ -180,7 +180,7 @@ class ClientManager: B.settings->>'race_id' as race_id, B.batch_size, B.num_nonces, - COALESCE(C.average_solution_quality, A.average_solution_quality) AS average_solution_quality, + COALESCE(C.average_quality, A.average_quality) AS average_quality, CASE WHEN B.end_time IS NOT NULL THEN 'COMPLETED' WHEN B.stopped IS NOT NULL THEN 'STOPPED' diff --git a/tig-benchmarker/master/master/job_manager.py b/tig-benchmarker/master/master/job_manager.py index 626b9e6e..eaa75aa3 100644 --- a/tig-benchmarker/master/master/job_manager.py +++ b/tig-benchmarker/master/master/job_manager.py @@ -28,7 +28,7 @@ class JobManager: algo_selection = CONFIG["algo_selection"] # create jobs from confirmed precommits challenge_id_2_name = { - c.id: c.details.name + c.id: c["config"]["name"] for c in challenges.values() } algorithm_id_2_name = { @@ -180,10 +180,10 @@ class JobManager: ( """ UPDATE job_data - SET average_solution_quality = %s - WHERE benchmark_id = %s + SET average_quality = %s + WHERE benchmark_id = %s AND average_quality IS NULL """, - (x.details.average_solution_quality, benchmark_id) + (sum(x.details.average_quality_by_bundle) // len(x.details.average_quality_by_bundle), benchmark_id) ) ] @@ -263,12 +263,12 @@ class JobManager: for row in rows: benchmark_id = row['benchmark_id'] solution_quality = [x for y in row['solution_quality'] for x in y] - average_solution_quality = sum(solution_quality) // len(solution_quality) + average_quality = sum(solution_quality) // len(solution_quality) batch_merkle_roots = [MerkleHash.from_str(root) for root in row['batch_merkle_roots']] num_batches = len(batch_merkle_roots) - logger.info(f"job {benchmark_id}: (benchmark ready, average_solution_nonces: {average_solution_quality}") + logger.info(f"job {benchmark_id}: (benchmark ready, average_solution_nonces: {average_quality}") tree = MerkleTree( batch_merkle_roots, @@ -283,13 +283,13 @@ class JobManager: UPDATE job_data SET merkle_root = %s, solution_quality = %s, - average_solution_quality = %s + average_quality = %s WHERE benchmark_id = %s """, ( merkle_root.to_str(), json.dumps(solution_quality), - average_solution_quality, + average_quality, benchmark_id ) ), diff --git a/tig-benchmarker/master/master/precommit_manager.py b/tig-benchmarker/master/master/precommit_manager.py index 10285463..8549308b 100644 --- a/tig-benchmarker/master/master/precommit_manager.py +++ b/tig-benchmarker/master/master/precommit_manager.py @@ -22,7 +22,7 @@ class PrecommitManager: self.last_block_id = block.id self.num_precommits_submitted = 0 - def run(self, race_samples: Dict[str, List[int]]) -> SubmitPrecommitRequest: + def run(self) -> SubmitPrecommitRequest: num_pending_jobs = get_db_conn().fetch_one( """ SELECT COUNT(*) @@ -49,11 +49,11 @@ class PrecommitManager: algorithm_id=a_id, player_id=CONFIG["player_id"], block_id=self.last_block_id, - race_id=race_samples[a_id] + track_id=selection["selected_track"] ), - num_nonces=selection["num_nonces"], + num_bundles=selection["num_bundles"], hyperparameters=selection["hyperparameters"], runtime_config=selection["runtime_config"], ) - logger.info(f"Created precommit (algorithm_id: {a_id}, race: {req.settings.race_id}, num_nonces: {req.num_nonces}, hyperparameters: {req.hyperparameters}, runtime_config: {req.runtime_config})") + logger.info(f"Created precommit (algorithm_id: {a_id}, track: {req.settings.track_id}, num_bundles: {req.num_bundles}, hyperparameters: {req.hyperparameters}, runtime_config: {req.runtime_config})") return req \ No newline at end of file diff --git a/tig-benchmarker/master/master/race_sampler.py b/tig-benchmarker/master/master/race_sampler.py deleted file mode 100644 index da3320f9..00000000 --- a/tig-benchmarker/master/master/race_sampler.py +++ /dev/null @@ -1,38 +0,0 @@ -import logging -import os -import random -from common.structs import * -from typing import List, Dict -from master.client_manager import CONFIG - -logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) - -Point = List[int] - -class RaceSampler: - def __init__(self): - self.valid_size_ranges = {} - - def on_new_block(self, block: Block, **kwargs): - self.active_race_ids = { - c_id: d["active_race_ids"] - for c_id, d in block.config["challenges"].items() - } - - def run(self) -> Dict[str, int]: - samples = {} - - for config in CONFIG["algo_selection"]: - a_id = config["algorithm_id"] - c_id = a_id[:4] - active_races = self.active_race_ids[c_id] - - selected_races = sorted(set(config["selected_races"]) & set(active_races)) - if len(selected_races) == 0: - selected_races = list(active_races) - config["selected_races"] = selected_races - - samples[a_id] = random.choice(selected_races) - logger.debug(f"Selected race '{samples[a_id]}' for algorithm {a_id} in challenge {c_id}") - - return samples \ No newline at end of file diff --git a/tig-benchmarker/master/master/slave_manager.py b/tig-benchmarker/master/master/slave_manager.py index da090c07..958190e3 100644 --- a/tig-benchmarker/master/master/slave_manager.py +++ b/tig-benchmarker/master/master/slave_manager.py @@ -241,7 +241,7 @@ class SlaveManager: UPDATE batch_data SET merkle_root = %s, solution_quality = %s, - average_solution_quality = %s + average_quality = %s WHERE benchmark_id = %s AND batch_idx = %s """, diff --git a/tig-benchmarker/master/master/submissions_manager.py b/tig-benchmarker/master/master/submissions_manager.py index 07293099..3d73d140 100644 --- a/tig-benchmarker/master/master/submissions_manager.py +++ b/tig-benchmarker/master/master/submissions_manager.py @@ -14,7 +14,7 @@ logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0]) @dataclass class SubmitPrecommitRequest(FromDict): settings: BenchmarkSettings - num_nonces: int + num_bundles: int hyperparameters: Optional[dict] runtime_config: dict diff --git a/tig-benchmarker/postgres/init.sql b/tig-benchmarker/postgres/init.sql index 23a121b4..d2340047 100644 --- a/tig-benchmarker/postgres/init.sql +++ b/tig-benchmarker/postgres/init.sql @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS job_data ( benchmark_id TEXT PRIMARY KEY, merkle_root TEXT, solution_quality JSONB, - average_solution_quality INTEGER, + average_quality INTEGER, merkle_proofs JSONB, FOREIGN KEY (benchmark_id) REFERENCES job(benchmark_id) @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS batch_data ( batch_idx INTEGER, merkle_root TEXT, solution_quality JSONB, - average_solution_quality INTEGER, + average_quality INTEGER, merkle_proofs JSONB, PRIMARY KEY (benchmark_id, batch_idx), @@ -115,8 +115,8 @@ SELECT ' "algo_selection": [ { "algorithm_id": "c001_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, @@ -127,8 +127,8 @@ SELECT ' }, { "algorithm_id": "c002_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, @@ -139,8 +139,8 @@ SELECT ' }, { "algorithm_id": "c003_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, @@ -151,8 +151,8 @@ SELECT ' }, { "algorithm_id": "c004_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, @@ -163,8 +163,8 @@ SELECT ' }, { "algorithm_id": "c005_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, @@ -175,8 +175,8 @@ SELECT ' }, { "algorithm_id": "c006_a001", - "num_nonces": 40, - "selected_races": [], + "num_bundles": 1, + "selected_track": null, "weight": 1, "batch_size": 8, "hyperparameters": null, diff --git a/tig-benchmarker/tests/data.py b/tig-benchmarker/tests/data.py index e8a1a6dc..6ca99695 100644 --- a/tig-benchmarker/tests/data.py +++ b/tig-benchmarker/tests/data.py @@ -27,7 +27,7 @@ class TestData(unittest.TestCase): block_id="some_block", challenge_id="some_challenge", algorithm_id="some_algorithm", - race_id="a=1,b=2" + track_id="a=1,b=2" ) rand_hash = "random_hash" @@ -35,8 +35,8 @@ class TestData(unittest.TestCase): # Assert same as Rust version: tig-structs/tests/core.rs expected = bytes([ - 122, 94, 247, 46, 146, 71, 140, 234, 78, 160, 235, 180, 79, 32, 69, 205, 247, 91, 94, - 43, 231, 184, 120, 114, 182, 226, 24, 176, 227, 170, 72, 31 + 84, 136, 44, 57, 142, 50, 248, 37, 94, 195, 254, 190, 222, 27, 136, 115, 229, 136, 19, + 207, 7, 208, 15, 193, 111, 99, 209, 131, 27, 189, 226, 175 ]) self.assertEqual(settings.calc_seed(rand_hash, nonce), expected) diff --git a/tig-benchmarker/ui/src/app/pages/home/home.component.html b/tig-benchmarker/ui/src/app/pages/home/home.component.html index 8b73ca1c..e6613156 100644 --- a/tig-benchmarker/ui/src/app/pages/home/home.component.html +++ b/tig-benchmarker/ui/src/app/pages/home/home.component.html @@ -88,8 +88,8 @@ ALGORITHM - - RACE + + TRACK BATCH SIZE @@ -97,7 +97,7 @@ #NONCES - AVG SOLUTION QUALITY + AVERAGE QUALITY STATUS @@ -142,7 +142,7 @@
- {{ benchmark.race_id_display }} + {{ benchmark.track_id_display }}
@@ -157,7 +157,7 @@
- {{ benchmark.average_solution_quality }} + {{ benchmark.average_quality }}
@@ -256,7 +256,7 @@
- {{ batch.end_time ? batch.average_solution_quality : "" }} + {{ batch.end_time ? batch.average_quality : "" }}
diff --git a/tig-benchmarker/ui/src/app/services/tig-apis.service.ts b/tig-benchmarker/ui/src/app/services/tig-apis.service.ts index d0e9d74b..c509938b 100644 --- a/tig-benchmarker/ui/src/app/services/tig-apis.service.ts +++ b/tig-benchmarker/ui/src/app/services/tig-apis.service.ts @@ -100,13 +100,13 @@ export class TigApisService { }; }); - const race_id_display = b.race_id + const track_id_display = b.track_id .replace(/=/g, ': ') .replace(/,/g, '\n'); return { ...b, - race_id_display, + track_id_display, time_elapsed, benchmark_id_display, batches,