coosld/tig-monorepo
1
0
Fork 0
mirror of https://github.com/tig-foundation/tig-monorepo.git synced 2026-02-21 10:27:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Save data in compressed form.

Browse Source
This commit is contained in:
FiveMovesAhead 2024-12-18 11:52:23 +08:00
parent cbabe083cf
commit 0c7526e709
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tig-benchmarker/slave.py
View File

@ -7,6 +7,7 @@ import requests
import shutil
import subprocess
import time
import zlib
from threading import Thread
from common.structs import OutputData, MerkleProof
from common.merkle_tree import MerkleTree
@ -98,10 +99,7 @@ def send_results(session, master_ip, master_port, tig_worker_path, download_wasm
if (
not os.path.exists(f"{output_folder}/result.json")
or not all(
os.path.exists(f"{output_folder}/{nonce}.json")
for nonce in range(batch["start_nonce"], batch["start_nonce"] + batch["num_nonces"])
)
or not os.path.exists(f"{output_folder}/data.zlib")
):
if os.path.exists(f"{output_folder}/result.json"):
os.remove(f"{output_folder}/result.json")
@ -128,8 +126,11 @@ def send_results(session, master_ip, master_port, tig_worker_path, download_wasm
time.sleep(2)
else:
with open(f"{output_folder}/data.json") as f:
leafs = [OutputData.from_dict(x) for x in json.load(f)]
with open(f"{output_folder}/data.zlib", "rb") as f:
leafs = [
OutputData.from_dict(x)
for x in json.loads(zlib.decompress(f.read()).decode())
]
merkle_tree = MerkleTree(
[x.to_merkle_hash() for x in leafs],

6
tig-worker/src/main.rs
View File

@ -5,7 +5,7 @@ use futures::stream::{self, StreamExt};
use serde_json::json;
use std::{fs, path::PathBuf, sync::Arc};
use tig_structs::core::BenchmarkSettings;
use tig_utils::{dejsonify, jsonify, MerkleHash, MerkleTree};
use tig_utils::{compress_obj, dejsonify, jsonify, MerkleHash, MerkleTree};
use tig_worker::OutputData;
use tokio::runtime::Runtime;
@ -272,8 +272,8 @@ fn compute_batch(
}
if let Some(path) = output_folder {
dump.sort_by_key(|data| data.nonce);
let file_path = path.join("data.json");
fs::write(&file_path, jsonify(&dump))?;
let file_path = path.join("data.zlib");
fs::write(&file_path, compress_obj(&dump))?;
}
let tree = MerkleTree::new(hashes, batch_size as usize)?;