From 76d2e3cfdf69ec1c6fb8a45cedaac42b29daacfa Mon Sep 17 00:00:00 2001 From: FiveMovesAhead Date: Tue, 6 Aug 2024 09:59:12 +0800 Subject: [PATCH] Simplify scripts. --- README.md | 5 +- scripts/get_benchmark_data.sh | 4 + scripts/list_algorithms.sh | 11 +- scripts/test_algorithm.sh | 102 ++++++++++++++++++ scripts/test_algorithm_performance.sh | 86 --------------- ...hmark_solutions.sh => verify_benchmark.sh} | 9 +- tig-worker/README.md | 35 ++++++ 7 files changed, 158 insertions(+), 94 deletions(-) create mode 100644 scripts/get_benchmark_data.sh create mode 100644 scripts/test_algorithm.sh delete mode 100644 scripts/test_algorithm_performance.sh rename scripts/{verify_benchmark_solutions.sh => verify_benchmark.sh} (91%) diff --git a/README.md b/README.md index 950bf22..af47d27 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,9 @@ Under `scripts/` folder is a bunch of useful bash scripts: * `list_algorithms.sh` * `list_benchmark_ids.sh` * `list_challenges.sh` -* `test_performance.sh` -* `verify_benchmark_solutions.sh` +* `get_benchmark_data.sh` +* `test_algorithm.sh` +* `verify_benchmark.sh` ## License diff --git a/scripts/get_benchmark_data.sh b/scripts/get_benchmark_data.sh new file mode 100644 index 0000000..fd4fd64 --- /dev/null +++ b/scripts/get_benchmark_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +read -p "Enter benchmark_id: " benchmark_id +curl -s https://mainnet-api.tig.foundation/get-benchmark-data?benchmark_id=$benchmark_id \ No newline at end of file diff --git a/scripts/list_algorithms.sh b/scripts/list_algorithms.sh index 8d2dcf3..bdbe2c8 100644 --- a/scripts/list_algorithms.sh +++ b/scripts/list_algorithms.sh @@ -9,9 +9,16 @@ WASMS_DICT=$(echo $RESP | jq -c '[.wasms[] | {key: .algorithm_id, value: .}] | f for ALGO in $(echo $ALGORITHMS | jq -c '.[]'); do ID=$(echo $ALGO | jq -r '.id') - NAME=$(echo $ALGO | jq -r '.details.name') + A_NAME=$(echo $ALGO | jq -r '.details.name') + case $(echo $ALGO | jq -r '.details.challenge_id') in + "c001") C_NAME="satisfiability" ;; + "c002") C_NAME="vehicle_routing" ;; + "c003") C_NAME="knapsack" ;; + "c004") C_NAME="vector_search" ;; + *) echo "unknown" ;; + esac ROUND_SUBMITTED=$(echo $ALGO | jq -r '.state.round_submitted') ROUND_PUSHED=$(echo $ALGO | jq -r '.state.round_pushed') COMPILE_SUCCESS=$(echo $WASMS_DICT | jq -c --arg ID "$ID" '.[$ID] | .details.compile_success') - printf "(%-9s) %-25s %-20s %-20s %-20s\n" "$ID" "$NAME" "round_submitted: $ROUND_SUBMITTED" "round_pushed: $ROUND_PUSHED" "compile_success: $COMPILE_SUCCESS" + printf "(%-9s) %-40s %-20s %-20s %-20s\n" "$ID" "$C_NAME/$A_NAME" "round_submitted: $ROUND_SUBMITTED" "round_pushed: $ROUND_PUSHED" "compile_success: $COMPILE_SUCCESS" done \ No newline at end of file diff --git a/scripts/test_algorithm.sh b/scripts/test_algorithm.sh new file mode 100644 index 0000000..46dff53 --- /dev/null +++ b/scripts/test_algorithm.sh @@ -0,0 +1,102 @@ +#!/bin/bash +REPO_DIR=$(dirname $(dirname "$(realpath "$0")")) +TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker" + +if [ ! -f $TIG_WORKER_PATH ]; then + echo "Error: tig-worker binary not found at ./target/release/tig-worker" + echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release" + exit 1 +fi + + +options=() +index=0 +echo "Available WASMs:" +for w in $(find $REPO_DIR/tig-algorithms/wasm -name '*.wasm'); do + a_name=$(basename $w .wasm) + c_name=$(basename $(dirname $w)) + echo "$index) $c_name/$a_name" + options+=("$c_name/$a_name") + index=$((index + 1)) +done +echo "Don't see the algorithm you're looking for? Can run: git pull origin / --no-edit" +read -p "Enter the index of the algorithm to test: " selected_index +if [[ $selected_index =~ ^[0-9]+$ ]] && [ "$selected_index" -ge 0 ] && [ "$selected_index" -lt "$index" ]; then + selected_option=${options[$selected_index]} + echo "Testing: $selected_option" +else + echo "Invalid index" + exit 1 +fi + +CHALLENGE=$(dirname $selected_option) +ALGORITHM=$(basename $selected_option) + +case $CHALLENGE in + satisfiability) + CHALLENGE_ID="c001" + ;; + vehicle_routing) + CHALLENGE_ID="c002" + ;; + knapsack) + CHALLENGE_ID="c003" + ;; + vector_search) + CHALLENGE_ID="c004" + ;; + *) + echo "Error: Challenge '$CHALLENGE' is not recognized." + exit 1 + ;; +esac + +read -p "Enter difficulty for $CHALLENGE in format [x,y]: " difficulty +read -p "Enter starting nonce: " start_nonce +read -p "Enter number of nonces: " num_nonces + +SETTINGS="{\"challenge_id\":\"$CHALLENGE_ID\",\"difficulty\":$difficulty,\"algorithm_id\":\"\",\"player_id\":\"\",\"block_id\":\"\"}" +num_solutions=0 +num_invalid=0 +num_errors=0 +total_ms=0 + +echo "----------------------------------------------------------------------" +echo "Testing performance of $CHALLENGE/$ALGORITHM" +echo "Settings: $SETTINGS" +echo "Starting nonce: $start_nonce" +echo "Number of nonces: $num_nonces" +echo -ne "" +for ((nonce=start_nonce; nonce&1) + exit_code=$? + end_time=$(date +%s%3N) + duration=$((end_time - start_time)) + total_ms=$((total_ms + duration)) + if [ $exit_code -eq 0 ]; then + num_solutions=$((num_solutions + 1)) + else + if echo "$output" | grep -q "Invalid solution\|No solution found"; then + num_invalid=$((num_invalid + 1)) + else + num_errors=$((num_errors + 1)) + fi + fi + if [ $((num_solutions)) -eq 0 ]; then + avg_ms_per_solution=0 + else + avg_ms_per_solution=$((total_ms / num_solutions)) + fi + echo -ne "#instances: $((num_solutions + num_invalid + num_errors)), #solutions: $num_solutions, #invalid: $num_invalid, #errors: $num_errors, average ms/solution: $avg_ms_per_solution\r" +done +echo +echo "----------------------------------------------------------------------" +echo "To re-run this test, run the following commands:" +echo " git clone https://github.com/tig-foundation/tig-monorepo.git" +echo " cd tig-monorepo" +echo " git checkout origin/$CHALLENGE/$ALGORITHM" +echo " bash scripts/test_algorithm_performance.sh" +echo "----------------------------------------------------------------------" +echo "Share your results on https://www.reddit.com/r/TheInnovationGame" +echo "----------------------------------------------------------------------" \ No newline at end of file diff --git a/scripts/test_algorithm_performance.sh b/scripts/test_algorithm_performance.sh deleted file mode 100644 index acc5263..0000000 --- a/scripts/test_algorithm_performance.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -set -e - -echo "Enter settings (JSON string):" -read settings - -echo "Enter starting nonce:" -read start_nonce - -echo "Enter number of nonces:" -read num_nonces - -challenge_id=$(echo "$settings" | jq -r '.challenge_id') -algorithm_id=$(echo "$settings" | jq -r '.algorithm_id') - -if [[ -z "$challenge_id" || "$challenge_id" == "null" ]]; then - echo "Error: Unable to parse challenge_id" - exit 1 -fi -if [[ -z "$algorithm_id" || "$algorithm_id" == "null" ]]; then - echo "Error: Unable to parse algorithm_id" - exit 1 -fi - -echo "Fetching names for challenge '$challenge_id' and algorithm '$algorithm_id'" -block_id=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id') -algorithm_name=$(curl -s "https://mainnet-api.tig.foundation/get-algorithms?block_id=$block_id" | jq -r --arg ID "$algorithm_id" '.algorithms[] | select(.id == $ID) | .details.name') -challenge_name=$(curl -s "https://mainnet-api.tig.foundation/get-challenges?block_id=$block_id" | jq -r --arg ID "$challenge_id" '.challenges[] | select(.id == $ID) | .details.name') -if [[ -z "$algorithm_name" || "$algorithm_name" == "null" ]]; then - echo "Error: Unable to find algorithm_name for '$algorithm_id'" - exit 1 -fi -if [[ -z "$challenge_name" || "$challenge_name" == "null" ]]; then - echo "Error: Unable to find challenge_name for '$challenge_id'" - exit 1 -fi - -branch="test_performance/${challenge_name}/${algorithm_name}" -if git show-ref --quiet $branch; then - echo "Branch $branch already exists. Switching" - git checkout $branch -else - echo "Create new branch with name $branch? (y/n)" - read confirm - if [ "$confirm" != "y" ]; then - echo "Aborting." - exit 1 - fi - git fetch origin - git checkout -b $branch origin/blank_slate -fi -git pull origin $challenge_name/$algorithm_name --no-edit - -cargo build -p tig-worker --release --features ${challenge_name}_${algorithm_name} - -solutions=0 -invalid=0 -total_ms=0 - -echo "------------------------------------------------------------" -echo "Testing performance of $challenge_name/$algorithm_name" -echo "Settings: $settings" -echo "Starting nonce: $start_nonce" -echo "Number of nonces: $num_nonces" -echo -ne "" -for ((nonce=start_nonce; nonce /dev/null 2>&1 - exit_code=$? - end_time=$(date +%s%3N) - duration=$((end_time - start_time)) - total_ms=$((total_ms + duration)) - if [ $exit_code -eq 0 ]; then - solutions=$((solutions + 1)) - else - invalid=$((invalid + 1)) - fi - if [ $((solutions)) -eq 0 ]; then - avg_ms_per_solution=0 - else - avg_ms_per_solution=$((total_ms / solutions)) - fi - echo -ne "#instances: $((solutions + invalid)), #solutions: $solutions, #invalid: $invalid, average ms/solution: $avg_ms_per_solution, average ms/nonce: $((total_ms / (solutions + invalid)))\r" -done -echo -echo "------------------------------------------------------------" \ No newline at end of file diff --git a/scripts/verify_benchmark_solutions.sh b/scripts/verify_benchmark.sh similarity index 91% rename from scripts/verify_benchmark_solutions.sh rename to scripts/verify_benchmark.sh index a3cd018..a8770b8 100644 --- a/scripts/verify_benchmark_solutions.sh +++ b/scripts/verify_benchmark.sh @@ -1,9 +1,12 @@ #!/bin/bash set -e -if [ ! -f ./target/release/tig-worker ]; then +REPO_DIR=$(dirname $(dirname "$(realpath "$0")")) +TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker" + +if [ ! -f $TIG_WORKER_PATH ]; then echo "Error: tig-worker binary not found at ./target/release/tig-worker" - echo "Run: cargo build -p tig-worker --release" + echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release" exit 1 fi @@ -64,8 +67,6 @@ for i in $(seq 0 $(($solutions_count - 1))); do echo "Ok" else echo "Mismatch. Actual: $compute_output" - rm $algorithm_id.wasm - exit 1 fi done rm $algorithm_id.wasm \ No newline at end of file diff --git a/tig-worker/README.md b/tig-worker/README.md index 0e9815a..5cd1057 100644 --- a/tig-worker/README.md +++ b/tig-worker/README.md @@ -62,6 +62,28 @@ Options: -h, --help Print help ``` +**Example:** +``` +SETTINGS='{"challenge_id":"c001","difficulty":[50,300],"algorithm_id":"","player_id":"","block_id":""}' +NONCE=0 +WASM=./tig-algorithms/wasm/satisfiability/sprint_sat.wasm +./target/release/tig-worker compute_solution $SETTINGS $NONCE $WASM +``` + +**Notes**: +* `challenge_id` must be set: + * `c001` is satisfiability + * `c002` is vehicle_routing + * `c003` is knapsack + * `c004` is vector_search +* Recommended low difficulties for testing are: + * satisfiability [50,300] + * vehicle_routing [40, 250] + * knapsack [50, 10] + * vector_search [10, 350] +* You can query the latest difficulties by using the `bash scripts/list_challenges.sh` +* You can test the performance of an algorithm using `bash scripts/test_algorithm.sh` + ## Verify Solution Given settings, nonce and a solution, `tig-worker` verifies the solution is a valid solution for the challenge instance. @@ -82,6 +104,19 @@ Options: -h, --help Print help ``` +**Example:** +``` +SETTINGS='{"challenge_id":"c001","difficulty":[50,300],"algorithm_id":"","player_id":"","block_id":""}' +NONCE=0 +SOLUTION='{"variables":[1,0,1,1,1,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,1,1,0,0,1,1,0,0,1,0,1,0,1,1,0,0,0,1,1,0,1,1,0]}' +./target/release/tig-worker verify_solution $SETTINGS $NONCE $SOLUTION +``` + +**Notes**: +* You can list all active benchmark ids with `scripts/list_benchmark_ids.sh` +* You get benchmark data with `scripts/list_benchmark_ids.sh` +* You verify a benchmark's solutions, runtime_signature and fuel_consumed with `scripts/verify_benchmark.sh` + # License [End User License Agreement](../docs/agreements/end_user_license_agreement.pdf) \ No newline at end of file