diff --git a/Dockerfile.dev b/Dockerfile.dev index 3c7448b..76dab15 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -8,8 +8,8 @@ RUN ARCH=$(uname -m) && \ echo "Unsupported architecture: $ARCH. Must be 'aarch64', 'arm64', 'amd64', or 'x86_64'." && exit 1; \ fi -# Common setup for all images -RUN apt update && apt install -y curl build-essential zstd python3 +RUN apt update && apt install -y curl build-essential zstd python3 python3-pip +RUN pip3 install blake3 requests --break-system-packages # Install Rust with specific version RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -58,8 +58,9 @@ RUN if command -v nvcc > /dev/null 2>&1; then \ chmod +x /usr/local/bin/tig-verifier && \ rm -rf tig-monorepo -COPY tig-binary/scripts /usr/local/bin/ -RUN chmod +x /usr/local/bin/build_so && \ - chmod +x /usr/local/bin/build_ptx +COPY scripts/ /usr/local/bin/tig-scripts/ +COPY tig-binary/scripts/ /usr/local/bin/tig-scripts/ +RUN chmod +x /usr/local/bin/tig-scripts/* +ENV PATH="/usr/local/bin/tig-scripts:${PATH}" WORKDIR /app diff --git a/Dockerfile.runtime b/Dockerfile.runtime index f3c2a62..3c84a2e 100644 --- a/Dockerfile.runtime +++ b/Dockerfile.runtime @@ -18,6 +18,10 @@ RUN chmod +x /usr/local/bin/tig-runtime && \ RUN apt update && apt install -y python3 python3-pip RUN pip3 install blake3 requests randomname --break-system-packages +COPY scripts/ /usr/local/bin/tig-scripts/ +RUN chmod +x /usr/local/bin/tig-scripts/* +ENV PATH="/usr/local/bin/tig-scripts:${PATH}" + COPY tig-benchmarker /app WORKDIR /app diff --git a/README.md b/README.md index 0712b03..96f437e 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,23 @@ This repository contains the implementation of The Innovation Game (TIG). * [tig-utils](./tig-utils/README.md) - A Rust crate that contains utility functions used throughout TIG * [tig-verifier](./tig-verifier/README.md) - A Rust crate that verifies a single solution or Merkle proof. +## Docker Images + +TIG docker images are hosted on [Github Packages](https://github.com/orgs/tig-foundation/packages): + +* [dev](https://github.com/orgs/tig-foundation/packages/container/package/tig-monorepo%2Fdev) - environment for Innovators who are developing algorithms +* [runtime](https://github.com/orgs/tig-foundation/packages/container/package/tig-monorepo%2Fruntime) - environment for Benchmarkers who are running slaves + ## Useful Scripts -Under `scripts/` folder is a bunch of useful bash scripts: +Under `scripts/` folder is a bunch of useful scripts: -* `list_algorithms.sh` -* `list_benchmark_ids.sh` -* `list_challenges.sh` -* `get_benchmark_data.sh` -* `test_algorithm.sh` -* `verify_benchmark.sh` +* `download_algorithm` +* `list_algorithms` +* `list_challenges` +* `test_algorithms` + +These are available on `PATH` in the `dev` and `runtime` docker images ## License diff --git a/scripts/download_algorithm b/scripts/download_algorithm new file mode 100644 index 0000000..3364665 --- /dev/null +++ b/scripts/download_algorithm @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +import argparse +import io +import os +import requests +import sys +import tarfile + +def main(): + parser = argparse.ArgumentParser(description='TIG Algorithm Downloader') + parser.add_argument('challenge', help="Challenge name or id") + parser.add_argument('algorithm', help="Algorithm name or id") + parser.add_argument('--out', default="tig-algorithms/lib", help="Output directory (default: tig-algorithms/lib)") + parser.add_argument('--testnet', action='store_true', help="Use testnet API") + + args = parser.parse_args() + + api_url = f"https://{'test' if args.testnet else 'main'}net-api.tig.foundation" + block = requests.get(f"{api_url}/get-block").json()["block"] + challenges = requests.get(f"{api_url}/get-challenges?block_id={block['id']}").json()["challenges"] + data = requests.get(f"{api_url}/get-algorithms?block_id={block['id']}").json() + algorithms = data["algorithms"] + download_urls = {x['algorithm_id']: x['details']['download_url'] for x in data['binarys']} + + c_id = next( + ( + c['id'] for c in challenges + if c['details']['name'] == args.challenge or c['id'] == args.challenge + ), + None + ) + if c_id is None: + print(f"Challenge '{args.challenge}' not found.") + sys.exit(1) + + a_id = next( + ( + a['id'] for a in algorithms + if ( + a['details']['name'] == args.algorithm or a['id'] == args.algorithm + ) and a['details']['challenge_id'] == c_id + ), + None + ) + if a_id is None: + print(f"Algorithm '{args.algorithm}' not found for challenge '{args.challenge}'.") + sys.exit(1) + + download_url = download_urls.get(a_id) + if download_url is None: + print(f"Download URL for algorithm '{args.algorithm}' not found.") + sys.exit(1) + + print(f"Downloading algorithm '{args.algorithm}' from {download_url}...") + resp = requests.get(download_url, stream=True) + output_dir = f"{args.out}/{args.challenge}" + os.makedirs(output_dir, exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(resp.content), mode='r:gz') as tar: + tar.extractall(path=output_dir) + print(f"Algorithm '{args.algorithm}' downloaded and extracted to '{output_dir}'.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/get_benchmark_data.sh b/scripts/get_benchmark_data.sh deleted file mode 100644 index fd4fd64..0000000 --- a/scripts/get_benchmark_data.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e -read -p "Enter benchmark_id: " benchmark_id -curl -s https://mainnet-api.tig.foundation/get-benchmark-data?benchmark_id=$benchmark_id \ No newline at end of file diff --git a/scripts/list_algorithm_adoption.sh b/scripts/list_algorithm_adoption.sh deleted file mode 100644 index 9f7cdb7..0000000 --- a/scripts/list_algorithm_adoption.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -echo "Fetching latest block..." -BLOCK_ID=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -echo "Fetching algorithms..." -RESP=$(curl -s "https://mainnet-api.tig.foundation/get-algorithms?block_id=$BLOCK_ID") -echo "Filtering algorithms..." -ALGORITHMS=$(echo $RESP | jq -c '.algorithms[]' | jq 'select(.block_data.adoption | length > 15)' | jq -s 'sort_by(.block_data.adoption|tonumber)' | jq 'reverse') - -declare -A CHALLENGES - -echo "Calculating adoption percentage..." -for ALGO in $(echo $ALGORITHMS | jq -c '.[]'); do - A_NAME=$(echo $ALGO | jq -r '.details.name') - ADOPTION=$(echo $ALGO | jq -r '.block_data.adoption') - - ADOPTION_PERCENT=$(awk "BEGIN {printf \"%.2f\", $ADOPTION / 10000000000000000}") - - case $(echo $ALGO | jq -r '.details.challenge_id') in - "c001") C_NAME="satisfiability" ;; - "c002") C_NAME="vehicle_routing" ;; - "c003") C_NAME="knapsack" ;; - "c004") C_NAME="vector_search" ;; - *) C_NAME="unknown" ;; - esac - - CHALLENGES[$C_NAME]+="$(printf " %-30s %-20s\n" "$A_NAME" "adoption: $ADOPTION_PERCENT%")\n" -done -echo "Printing..." -echo "" -for CHALLENGE in "${!CHALLENGES[@]}"; do - echo " Challenge: $CHALLENGE" - echo -e "${CHALLENGES[$CHALLENGE]}" -done - diff --git a/scripts/list_algorithms b/scripts/list_algorithms new file mode 100644 index 0000000..aa5853c --- /dev/null +++ b/scripts/list_algorithms @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import argparse +import io +import os +import requests +import sys +import tarfile + +def main(): + parser = argparse.ArgumentParser(description='TIG Algorithm Lister') + parser.add_argument('challenge', help="Challenge name or id") + parser.add_argument('--testnet', action='store_true', help="Use testnet API") + + args = parser.parse_args() + + api_url = f"https://{'test' if args.testnet else 'main'}net-api.tig.foundation" + block = requests.get(f"{api_url}/get-block").json()["block"] + challenges = requests.get(f"{api_url}/get-challenges?block_id={block['id']}").json()["challenges"] + data = requests.get(f"{api_url}/get-algorithms?block_id={block['id']}").json() + algorithms = data["algorithms"] + compile_success = {x['algorithm_id']: x['details']['compile_success'] for x in data['binarys']} + + c_id = next( + ( + c['id'] for c in challenges + if c['details']['name'] == args.challenge or c['id'] == args.challenge + ), + None + ) + if c_id is None: + print(f"Challenge '{args.challenge}' not found.") + sys.exit(1) + + algorithms = sorted([ + a for a in algorithms if a['details']['challenge_id'] == c_id + ], key=lambda x: x['id']) + + for a in algorithms: + if a["id"] not in compile_success: + status = f"pending compilation" + elif not compile_success[a["id"]]: + status = f"failed to compile" + elif a["state"]["round_merged"] is not None: + status = f"merged @ round {a['state']['round_merged']}" + elif a["state"]["round_active"] <= block["details"]["round"]: + status = f"active with {a['block_data']['merge_points']} merge points" + elif a["state"]["round_pushed"] <= block["details"]["round"]: + status = f"active @ round {a['state']['round_active']}" + else: + status = f"pushed @ round {a['state']['round_pushed']}" + print(f"id: {a['id']:<12} name: {a['details']['name']:<20} status: {status}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/list_algorithms.sh b/scripts/list_algorithms.sh deleted file mode 100644 index a4c9d74..0000000 --- a/scripts/list_algorithms.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e - -BLOCK_ID=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -RESP=$(curl -s "https://mainnet-api.tig.foundation/get-algorithms?block_id=$BLOCK_ID") - -ALGORITHMS=$(echo $RESP | jq -c '.algorithms[]' | jq -s 'sort_by(.id)') -WASMS_DICT=$(echo $RESP | jq -c '[.binarys[] | {key: .algorithm_id, value: .}] | from_entries') - -for ALGO in $(echo $ALGORITHMS | jq -c '.[]'); do - ID=$(echo $ALGO | jq -r '.id') - A_NAME=$(echo $ALGO | jq -r '.details.name') - case $(echo $ALGO | jq -r '.details.challenge_id') in - "c001") C_NAME="satisfiability" ;; - "c002") C_NAME="vehicle_routing" ;; - "c003") C_NAME="knapsack" ;; - "c004") C_NAME="vector_search" ;; - *) echo "unknown" ;; - esac - ROUND_SUBMITTED=$(echo $ALGO | jq -r '.state.round_submitted') - ROUND_PUSHED=$(echo $ALGO | jq -r '.state.round_pushed') - COMPILE_SUCCESS=$(echo $WASMS_DICT | jq -c --arg ID "$ID" '.[$ID] | .details.compile_success') - printf "(%-9s) %-40s %-20s %-20s %-20s\n" "$ID" "$C_NAME/$A_NAME" "round_submitted: $ROUND_SUBMITTED" "round_pushed: $ROUND_PUSHED" "compile_success: $COMPILE_SUCCESS" -done \ No newline at end of file diff --git a/scripts/list_block_benchmark_ids.sh b/scripts/list_block_benchmark_ids.sh deleted file mode 100644 index 697dbf0..0000000 --- a/scripts/list_block_benchmark_ids.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -e -curl -s https://mainnet-api.tig.foundation/get-block?include_data | jq -r '.block.data.active_ids.benchmark[]' | nl \ No newline at end of file diff --git a/scripts/list_challenges b/scripts/list_challenges new file mode 100644 index 0000000..e716eaf --- /dev/null +++ b/scripts/list_challenges @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import argparse +import io +import os +import requests +import sys +import tarfile + +def main(): + parser = argparse.ArgumentParser(description='TIG Challenge Lister') + parser.add_argument('--testnet', action='store_true', help="Use testnet API") + + args = parser.parse_args() + + api_url = f"https://{'test' if args.testnet else 'main'}net-api.tig.foundation" + block = requests.get(f"{api_url}/get-block").json()["block"] + challenges = requests.get(f"{api_url}/get-challenges?block_id={block['id']}").json()["challenges"] + + challenges = sorted(challenges, key=lambda x: x['id']) + for c in challenges: + status = f"active @ round {c['state']['round_active']}" + print(f"id: {c['id']:<7} name: {c['details']['name']:<20} status: {status}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/list_challenges.sh b/scripts/list_challenges.sh deleted file mode 100644 index ec28bd8..0000000 --- a/scripts/list_challenges.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -BLOCK_ID=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -RESP=$(curl -s "https://mainnet-api.tig.foundation/get-challenges?block_id=$BLOCK_ID") - -CHALLENGES=$(echo $RESP | jq -c '.challenges[]' | jq -s 'sort_by(.id)') - -for C in $(echo $CHALLENGES | jq -c '.[]'); do - ID=$(echo $C | jq -r '.id') - NAME=$(echo $C | jq -r '.details.name') - ROUND_ACTIVE=$(echo $C | jq -r '.state.round_active') - NUM_QUALIFIERS=$(echo $C | jq -c '.block_data.num_qualifiers') - DIFFICULTIES=$(echo $C | jq -c '.block_data.qualifier_difficulties') - printf "(%-4s) %-20s %-20s %-20s\n%-s\n\n" "$ID" "$NAME" "round_active: $ROUND_ACTIVE" "num_qualifiers: $NUM_QUALIFIERS" "qualifier_difficulties: $DIFFICULTIES" -done \ No newline at end of file diff --git a/scripts/list_player_benchmarks.sh b/scripts/list_player_benchmarks.sh deleted file mode 100644 index 4407ed6..0000000 --- a/scripts/list_player_benchmarks.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -e - -# Ask for player_id as input -read -p "Enter player ID: " PLAYER_ID - -BLOCK_ID=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -RESP=$(curl -s "https://mainnet-api.tig.foundation/get-benchmarks?block_id=$BLOCK_ID&player_id=$PLAYER_ID") - -BENCHMARKS=$(echo $RESP | jq -c '[.benchmarks[]] | sort_by(.settings.challenge_id, -.details.num_solutions)') - -declare -A SOLUTIONS_COUNT - -for BENCHMARK in $(echo $BENCHMARKS | jq -c '.[]'); do - ID=$(echo $BENCHMARK | jq -r '.id') - SETTINGS=$(echo $BENCHMARK | jq -c '.settings') - CHALLENGE_ID=$(echo $BENCHMARK | jq -r '.settings.challenge_id') - NUM_SOLUTIONS=$(echo $BENCHMARK | jq -r '.details.num_solutions') - printf "ID: %-38s #Solutions: %-5s Settings: %-50s \n" "$ID" "$NUM_SOLUTIONS" "$SETTINGS" - SOLUTIONS_COUNT["$CHALLENGE_ID"]=$(( ${SOLUTIONS_COUNT["$CHALLENGE_ID"]:-0} + NUM_SOLUTIONS )) -done - -echo "Total solutions by Challenge ID:" -for CHALLENGE_ID in "${!SOLUTIONS_COUNT[@]}"; do - echo "Challenge ID: $CHALLENGE_ID, Total Solutions: ${SOLUTIONS_COUNT[$CHALLENGE_ID]}" -done \ No newline at end of file diff --git a/scripts/list_players.sh b/scripts/list_players.sh deleted file mode 100644 index 6f4f56e..0000000 --- a/scripts/list_players.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -BLOCK_ID=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -RESP=$(curl -s "https://mainnet-api.tig.foundation/get-opow?block_id=$BLOCK_ID") - -PLAYERS=$(echo $RESP | jq -c '[.opow[] | .block_data.reward = (if .block_data.reward == null then 0 else (.block_data.reward | tonumber) end)] | sort_by(.block_data.reward) | reverse') - -for PLAYER in $(echo $PLAYERS | jq -c '.[]'); do - ID=$(echo $PLAYER | jq -r '.player_id') - REWARD=$(echo $PLAYER | jq -r '.block_data.reward | if . == null then "null" else tonumber / 1e18 end') - printf "Player ID: %-25s Reward: %-20s\n" "$ID" "$REWARD" -done \ No newline at end of file diff --git a/scripts/test_algorithm b/scripts/test_algorithm new file mode 100644 index 0000000..5f77c9e --- /dev/null +++ b/scripts/test_algorithm @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +if (CPU_ARCH := platform.machine().lower()) in ["x86_64", "amd64"]: + CPU_ARCH = "amd64" +elif CPU_ARCH in ["arm64", "aarch64"]: + CPU_ARCH = "aarch64" +else: + print(f"Unsupported CPU architecture: {CPU_ARCH}") + sys.exit(1) + +HAS_GPU = subprocess.run(["which", "nvidia-smi"], capture_output=True).returncode == 0 +if (VISIBLE_CPUS := os.environ.get("CPU_VISIBLE_CORES", None)) is None: + VISIBLE_CPUS = list(os.sched_getaffinity(0)) +else: + VISIBLE_CPUS = list(map(int, VISIBLE_CPUS.split(","))) + os.sched_setaffinity(0, VISIBLE_CPUS) + +if not HAS_GPU: + VISIBLE_GPUS = [] +elif (VISIBLE_GPUS := os.environ.get("CUDA_VISIBLE_DEVICES", None)) is None: + VISIBLE_GPUS = [ + int(match.group(1)) + for line in subprocess.check_output(["nvidia-smi", "-L"]).decode("utf-8").splitlines() + if (match := re.match(r'^GPU (\d+):', line)) is not None + ] +else: + VISIBLE_GPUS = list(map(int, VISIBLE_GPUS.split(","))) + +def now(): + return int(time.time() * 1000) + + +if __name__ == "__main__": + tig_runtime_path = shutil.which("tig-runtime") + parser = argparse.ArgumentParser(description="TIG Algorithm Tester") + parser.add_argument("challenge", type=str, help="Challenge name") + parser.add_argument("algorithm", type=str, help="Algorithm name") + parser.add_argument("difficulty", type=str, help="JSON string of difficulty") + parser.add_argument("--tig_runtime_path", type=str, default=tig_runtime_path, help=f"Path to tig-runtime executable (default: {tig_runtime_path})") + parser.add_argument("--lib-dir", type=str, default="./tig-algorithms/lib", help="Path to the algorithms library folder (default: ./tig-algorithms/lib)") + parser.add_argument("--seed", type=str, default="rand_hash", help="String to use as seed instance generation (default: 'rand_hash')") + parser.add_argument("--start", type=int, default=0, help="Starting nonce (default: 0)") + parser.add_argument("--nonces", type=int, default=100, help="Number of nonces to process (default: 100)") + parser.add_argument("--fuel", type=int, default=int(100e9), help="Max fuel (default: 100 billion)") + parser.add_argument("--workers", type=int, default=1, help="Number of worker threads (default: 1)") + parser.add_argument("--verbose", action='store_true', help="Print debug logs") + + args = parser.parse_args() + + so_path = f"{args.lib_dir}/{args.challenge}/{CPU_ARCH}/{args.algorithm}.so" + ptx_path = f"{args.lib_dir}/{args.challenge}/ptx/{args.algorithm}.ptx" + + if not os.path.exists(so_path): + print( +f"""Library not found at {so_path}: + * To download: use download_algorithm + * To build: use build_so and/or build_ptx + * To set the lib folder: set --lib-dir +""") + sys.exit(1) + + if not os.path.exists(ptx_path): + ptx_path = None + elif not HAS_GPU: + print(f"PTX file found at {ptx_path}, but no GPU support detected (failed to run nvidia-smi)") + sys.exit(1) + + difficulty = json.loads(args.difficulty) + if not ( + isinstance(difficulty, list) and + len(difficulty) == 2 and + all(isinstance(x, int) for x in difficulty) + ): + print("Difficulty must be a JSON array of two integers '[x,y]'") + sys.exit(1) + + challenge_ids = { + "satisfiability": "c001", + "vehicle_routing": "c002", + "knapsack": "c003", + "vector_search": "c004", + "hypergraph": "c005", + "optimiser": "c006", + } + challenge_id = challenge_ids.get(args.challenge, None) + if challenge_id is None: + print(f"Unknown challenge '{args.challenge}'. Choices: ({', '.join(challenge_ids.keys())})") + sys.exit(1) + + settings = {"algorithm_id": "", "challenge_id": challenge_id, "difficulty": difficulty, "block_id": "", "player_id": ""} + pool = ThreadPoolExecutor(max_workers=args.workers + 1) + + results = {} + def print_results(): + start = now() + while True: + time.sleep(0.5) + num_processing, num_finished, num_solutions = 0, 0, 0 + for (_, _, ret) in results.values(): + if ret is None: + num_processing += 1 + else: + num_finished += 1 + num_solutions += int(ret == 0) + + elapsed = (now() - start) / 1000 + solution_ratio = num_solutions / (num_finished or 1) + solution_rate = num_solutions / elapsed + score = solution_rate * solution_ratio + out = f"#processing: {num_processing}, #finished: {num_finished}, #solutions: {num_solutions}, elapsed: {elapsed:.2f}s, solution_ratio: {solution_ratio:.4f}, solution_rate: {solution_rate:.4f}, score: {score:.4f}" + if args.verbose: + print(out) + else: + print(f"\r{out}", end="") + + if num_finished == args.nonces: + break + if not args.verbose: + print("\n") + + def run_tig_runtime(nonce): + cmd = [ + args.tig_runtime_path, + json.dumps(settings), + args.seed, + str(nonce), + so_path, + "--fuel", str(args.fuel), + ] + if ptx_path is not None: + cmd += [ + "--ptx", ptx_path, + "--gpu", str(nonce % len(VISIBLE_GPUS)), + ] + if args.verbose: + print(f"computing nonce {nonce}: {' '.join(cmd)}") + start = now() + results[nonce] = (start, None, None) + ret = subprocess.run(cmd, capture_output=True, text=True) + elapsed = now() - start + results[nonce] = (start, elapsed, ret.returncode) + if args.verbose: + out = f"computing nonce {nonce}: took {elapsed}ms, " + if ret.returncode == 0: + out += "found solution" + elif ret.returncode == 1: + out += f"error: {ret.stderr.strip()}" + elif ret.returncode == 85: + out += "no solution found" + elif ret.returncode == 86: + out += f"invalid solution: {ret.stderr.strip()}" + elif ret.returncode == 87: + out += "out of fuel" + else: + out += f"unknown exit code {ret.returncode}" + print(out) + + nonces = list(range(args.start, args.start + args.nonces)) + if args.verbose: + print(f"Processing {len(nonces)} nonces with {args.workers} workers...") + pool.submit(print_results) + list(pool.map(run_tig_runtime, nonces)) diff --git a/scripts/test_algorithm.sh b/scripts/test_algorithm.sh deleted file mode 100644 index edc2fdf..0000000 --- a/scripts/test_algorithm.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -REPO_DIR=$(dirname $(dirname "$(realpath "$0")")) -TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker" - -if [ ! -f $TIG_WORKER_PATH ]; then - echo "Error: tig-worker binary not found at ./target/release/tig-worker" - echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release" - exit 1 -fi - - -options=() -index=0 -echo "Available WASMs:" -for w in $(find $REPO_DIR/tig-algorithms/wasm -name '*.wasm'); do - a_name=$(basename $w .wasm) - c_name=$(basename $(dirname $w)) - echo "$index) $c_name/$a_name" - options+=("$c_name/$a_name") - index=$((index + 1)) -done -echo "Don't see the algorithm you're looking for? Can run: git pull origin / --no-edit" -read -p "Enter the index of the algorithm to test: " selected_index -if [[ $selected_index =~ ^[0-9]+$ ]] && [ "$selected_index" -ge 0 ] && [ "$selected_index" -lt "$index" ]; then - selected_option=${options[$selected_index]} - echo "Testing: $selected_option" -else - echo "Invalid index" - exit 1 -fi - -CHALLENGE=$(dirname $selected_option) -ALGORITHM=$(basename $selected_option) - -case $CHALLENGE in - satisfiability) - CHALLENGE_ID="c001" - ;; - vehicle_routing) - CHALLENGE_ID="c002" - ;; - knapsack) - CHALLENGE_ID="c003" - ;; - vector_search) - CHALLENGE_ID="c004" - ;; - *) - echo "Error: Challenge '$CHALLENGE' is not recognized." - exit 1 - ;; -esac - -read -p "Enter difficulty for $CHALLENGE in format [x,y]: " difficulty -regex='^\[[0-9]+,[0-9]+\]$' -if ! [[ $difficulty =~ $regex ]]; then - echo "Error: Difficulty must be in the format [x,y] where x and y are positive integers." - exit 1 -fi -is_positive_integer() { - [[ $1 =~ ^[0-9]+$ ]] && [ "$1" -ge 0 ] -} -read -p "Enter starting nonce: " start_nonce -if ! is_positive_integer "$start_nonce"; then - echo "Error: Starting nonce must be a positive integer." - exit 1 -fi -read -p "Enter number of nonces: " num_nonces -if ! is_positive_integer "$num_nonces"; then - echo "Error: Number of nonces must be a positive integer." - exit 1 -fi -read -p "Enter number of workers (default is 1): " num_workers -num_workers=${num_workers:-1} -if ! is_positive_integer "$num_workers"; then - echo "Error: Number of workers must be a positive integer." - exit 1 -fi -read -p "Enter max fuel (default is 10000000000): " max_fuel -max_fuel=${max_fuel:-10000000000} -if ! is_positive_integer "$max_fuel"; then - echo "Error: Max fuel must be a positive integer." - exit 1 -fi -read -p "Enable debug mode? (leave blank to disable) " enable_debug -if [[ -n $enable_debug ]]; then - debug_mode=true -else - debug_mode=false -fi - -get_closest_power_of_2() { - local n=$1 - local p=1 - while [ $p -lt $n ]; do - p=$((p * 2)) - done - echo $p -} - - -SETTINGS="{\"challenge_id\":\"$CHALLENGE_ID\",\"difficulty\":$difficulty,\"algorithm_id\":\"\",\"player_id\":\"\",\"block_id\":\"\"}" -num_solutions=0 -num_invalid=0 -total_ms=0 - -echo "----------------------------------------------------------------------" -echo "Testing performance of $CHALLENGE/$ALGORITHM" -echo "Settings: $SETTINGS" -echo "Starting nonce: $start_nonce" -echo "Number of nonces: $num_nonces" -echo "Number of workers: $num_workers" -echo -ne "" - -remaining_nonces=$num_nonces -current_nonce=$start_nonce - -while [ $remaining_nonces -gt 0 ]; do - nonces_to_compute=$((num_workers < remaining_nonces ? num_workers : remaining_nonces)) - - power_of_2_nonces=$(get_closest_power_of_2 $nonces_to_compute) - - start_time=$(date +%s%3N) - stdout=$(mktemp) - stderr=$(mktemp) - ./target/release/tig-worker compute_batch "$SETTINGS" "random_string" $current_nonce $nonces_to_compute $power_of_2_nonces $REPO_DIR/tig-algorithms/wasm/$CHALLENGE/$ALGORITHM.wasm --workers $nonces_to_compute --fuel $max_fuel >"$stdout" 2>"$stderr" - exit_code=$? - output_stdout=$(cat "$stdout") - output_stderr=$(cat "$stderr") - end_time=$(date +%s%3N) - duration=$((end_time - start_time)) - total_ms=$((total_ms + (duration * nonces_to_compute))) - - if [ $exit_code -eq 0 ]; then - solutions_count=$(echo "$output_stdout" | grep -o '"solution_nonces":\[.*\]' | sed 's/.*\[\(.*\)\].*/\1/' | awk -F',' '{print NF}') - invalid_count=$((nonces_to_compute - solutions_count)) - num_solutions=$((num_solutions + solutions_count)) - num_invalid=$((num_invalid + invalid_count)) - fi - - if [ $num_solutions -eq 0 ]; then - avg_ms_per_solution=0 - else - avg_ms_per_solution=$((total_ms / num_solutions)) - fi - - if [[ $debug_mode == true ]]; then - echo " Current nonce: $current_nonce" - echo " Nonces computed: $nonces_to_compute" - echo " Exit code: $exit_code" - echo " Stdout: $output_stdout" - echo " Stderr: $output_stderr" - echo " Duration: $duration ms" - echo "#instances: $((num_solutions + num_invalid)), #solutions: $num_solutions, #invalid: $num_invalid, average ms/solution: $avg_ms_per_solution" - else - echo -ne "#instances: $((num_solutions + num_invalid)), #solutions: $num_solutions, #invalid: $num_invalid, average ms/solution: $avg_ms_per_solution\033[K\r" - fi - - current_nonce=$((current_nonce + nonces_to_compute)) - remaining_nonces=$((remaining_nonces - nonces_to_compute)) -done - -echo -echo "----------------------------------------------------------------------" -echo "To re-run this test, run the following commands:" -echo " git clone https://github.com/tig-foundation/tig-monorepo.git" -echo " cd tig-monorepo" -echo " git pull origin/$CHALLENGE/$ALGORITHM --no-edit" -echo " bash scripts/test_algorithm.sh" -echo "----------------------------------------------------------------------" -echo "Share your results on https://www.reddit.com/r/TheInnovationGame" -echo "----------------------------------------------------------------------" -rm "$stdout" "$stderr" \ No newline at end of file diff --git a/scripts/verify_benchmark.sh b/scripts/verify_benchmark.sh deleted file mode 100644 index 0de1b6c..0000000 --- a/scripts/verify_benchmark.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -set -e - -REPO_DIR=$(dirname $(dirname "$(realpath "$0")")) -TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker" - -if [ ! -f $TIG_WORKER_PATH ]; then - echo "Error: tig-worker binary not found at ./target/release/tig-worker" - echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release" - exit 1 -fi - -read -p "Enter benchmark_id: " benchmark_id - -echo "Fetching benchmark data" -response=$(curl -s "https://mainnet-api.tig.foundation/get-benchmark-data?benchmark_id=$benchmark_id") - -# parse data from resp -proof=$(echo "$response" | python3 -c " -import sys, json -data = json.load(sys.stdin) -for item in data['proof']['solutions_data']: - item['nonce'] = str(item['nonce']) -print(json.dumps(data)) -" | jq '.proof') -if [ "$proof" == "null" ]; then - echo "No proofs found" - exit 0 -fi -solutions_count=$(echo "$proof" | jq -r '.solutions_data | length') -settings=$(echo "$response" | jq -r '.benchmark.settings') -algorithm_id=$(echo "$settings" | jq -r '.algorithm_id') -echo "Found $solutions_count solutions to verify" - -# Fetch block id -echo "Fetching block data" -block_response=$(curl -s "https://mainnet-api.tig.foundation/get-block") -block_id=$(echo "$block_response" | jq -r '.block.id') - -# Fetch algorithms for the block -echo "Fetching algorithms data" -algorithms_response=$(curl -s "https://mainnet-api.tig.foundation/get-algorithms?block_id=$block_id") -wasms=$(echo "$algorithms_response" | jq -c '.wasms[]') - -wasm=$(echo "$wasms" | jq -c "select(.algorithm_id == \"$algorithm_id\")") - -if [ -z "$wasm" ]; then - echo "No matching WASM found for $algorithm_id" - exit 0 -fi - -compile_success=$(echo "$wasm" | jq -r '.details.compile_success') -if [ "$compile_success" == "false" ]; then - echo "WASM was not successful compiled" - exit 0 -fi - -download_url=$(echo "$wasm" | jq -r '.details.download_url') -echo "Downloading WASM from $download_url" -curl -s -o "$algorithm_id.wasm" "$download_url" - -# verify solutions -for i in $(seq 0 $(($solutions_count - 1))); do - solution_data=$(echo "$proof" | jq -c -S ".solutions_data[$i]") - echo Verifying $solution_data - nonce=$(echo "$solution_data" | jq -r '.nonce') - solution=$(echo "$solution_data" | jq -r '.solution') - echo Verifying solution is valid - ./target/release/tig-worker verify_solution "$settings" $nonce "$solution" - echo Verifying runtime_signature and fuel_consumed - compute_output=$(./target/release/tig-worker compute_solution "$settings" $nonce ./$algorithm_id.wasm | python3 -c " -import sys, json -data = json.load(sys.stdin) -data['nonce'] = str(data['nonce']) -print(json.dumps(data)) - " | jq -c -S) - if [[ "$compute_output" == "$solution_data" ]]; then - echo "Ok" - else - echo "Mismatch. Actual: $compute_output" - fi -done -rm $algorithm_id.wasm \ No newline at end of file