Simplify scripts.

2026-02-21 17:37:21 +08:00 · 2024-08-06 09:59:12 +08:00 · 2024-08-06 09:59:12 +08:00 · 76d2e3cfdf
commit 76d2e3cfdf
parent 6956386a2e
7 changed files with 158 additions and 94 deletions
--- a/README.md
+++ b/README.md
@ -30,8 +30,9 @@ Under `scripts/` folder is a bunch of useful bash scripts:
 * `list_algorithms.sh`
 * `list_benchmark_ids.sh`
 * `list_challenges.sh`
-* `test_performance.sh`
-* `verify_benchmark_solutions.sh`
+* `get_benchmark_data.sh`
+* `test_algorithm.sh`
+* `verify_benchmark.sh`

 ## License

--- a/scripts/get_benchmark_data.sh
+++ b/scripts/get_benchmark_data.sh
@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+read -p "Enter benchmark_id: " benchmark_id
+curl -s https://mainnet-api.tig.foundation/get-benchmark-data?benchmark_id=$benchmark_id
--- a/scripts/list_algorithms.sh
+++ b/scripts/list_algorithms.sh
@ -9,9 +9,16 @@ WASMS_DICT=$(echo $RESP | jq -c '[.wasms[] | {key: .algorithm_id, value: .}] | f

 for ALGO in $(echo $ALGORITHMS | jq -c '.[]'); do
    ID=$(echo $ALGO | jq -r '.id')
-    NAME=$(echo $ALGO | jq -r '.details.name')
+    A_NAME=$(echo $ALGO | jq -r '.details.name')
+    case $(echo $ALGO | jq -r '.details.challenge_id') in
+        "c001") C_NAME="satisfiability" ;;
+        "c002") C_NAME="vehicle_routing" ;;
+        "c003") C_NAME="knapsack" ;;
+        "c004") C_NAME="vector_search" ;;
+        *) echo "unknown" ;;
+    esac
    ROUND_SUBMITTED=$(echo $ALGO | jq -r '.state.round_submitted')
    ROUND_PUSHED=$(echo $ALGO | jq -r '.state.round_pushed')
    COMPILE_SUCCESS=$(echo $WASMS_DICT | jq -c --arg ID "$ID" '.[$ID] | .details.compile_success')
-    printf "(%-9s) %-25s %-20s %-20s %-20s\n" "$ID" "$NAME" "round_submitted: $ROUND_SUBMITTED" "round_pushed: $ROUND_PUSHED" "compile_success: $COMPILE_SUCCESS"
+    printf "(%-9s) %-40s %-20s %-20s %-20s\n" "$ID" "$C_NAME/$A_NAME" "round_submitted: $ROUND_SUBMITTED" "round_pushed: $ROUND_PUSHED" "compile_success: $COMPILE_SUCCESS"
 done
--- a/scripts/test_algorithm.sh
+++ b/scripts/test_algorithm.sh
@ -0,0 +1,102 @@
+#!/bin/bash
+REPO_DIR=$(dirname $(dirname "$(realpath "$0")"))
+TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker"
+
+if [ ! -f $TIG_WORKER_PATH ]; then
+    echo "Error: tig-worker binary not found at ./target/release/tig-worker"
+    echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release"
+    exit 1
+fi
+
+
+options=()
+index=0
+echo "Available WASMs:"
+for w in $(find $REPO_DIR/tig-algorithms/wasm -name '*.wasm'); do
+    a_name=$(basename $w .wasm)
+    c_name=$(basename $(dirname $w))
+    echo "$index) $c_name/$a_name"
+    options+=("$c_name/$a_name")
+    index=$((index + 1))
+done
+echo "Don't see the algorithm you're looking for? Can run: git pull origin <challenge_name>/<algorithm_name> --no-edit"
+read -p "Enter the index of the algorithm to test: " selected_index
+if [[ $selected_index =~ ^[0-9]+$ ]] && [ "$selected_index" -ge 0 ] && [ "$selected_index" -lt "$index" ]; then
+    selected_option=${options[$selected_index]}
+    echo "Testing: $selected_option"
+else
+    echo "Invalid index"
+    exit 1
+fi
+
+CHALLENGE=$(dirname $selected_option)
+ALGORITHM=$(basename $selected_option)
+
+case $CHALLENGE in
+    satisfiability)
+        CHALLENGE_ID="c001"
+        ;;
+    vehicle_routing)
+        CHALLENGE_ID="c002"
+        ;;
+    knapsack)
+        CHALLENGE_ID="c003"
+        ;;
+    vector_search)
+        CHALLENGE_ID="c004"
+        ;;
+    *)
+        echo "Error: Challenge '$CHALLENGE' is not recognized."
+        exit 1
+        ;;
+esac
+
+read -p "Enter difficulty for $CHALLENGE in format [x,y]: " difficulty
+read -p "Enter starting nonce: " start_nonce
+read -p "Enter number of nonces: " num_nonces
+
+SETTINGS="{\"challenge_id\":\"$CHALLENGE_ID\",\"difficulty\":$difficulty,\"algorithm_id\":\"\",\"player_id\":\"\",\"block_id\":\"\"}"
+num_solutions=0
+num_invalid=0
+num_errors=0
+total_ms=0
+
+echo "----------------------------------------------------------------------"
+echo "Testing performance of $CHALLENGE/$ALGORITHM"
+echo "Settings: $SETTINGS"
+echo "Starting nonce: $start_nonce"
+echo "Number of nonces: $num_nonces"
+echo -ne ""
+for ((nonce=start_nonce; nonce<start_nonce+num_nonces; nonce++)); do
+    start_time=$(date +%s%3N)
+    output=$(./target/release/tig-worker compute_solution "$SETTINGS" $nonce $REPO_DIR/tig-algorithms/wasm/$CHALLENGE/$ALGORITHM.wasm 2>&1)
+    exit_code=$?
+    end_time=$(date +%s%3N)
+    duration=$((end_time - start_time))
+    total_ms=$((total_ms + duration))
+    if [ $exit_code -eq 0 ]; then
+        num_solutions=$((num_solutions + 1))
+    else
+      if echo "$output" | grep -q "Invalid solution\|No solution found"; then
+          num_invalid=$((num_invalid + 1))
+      else
+          num_errors=$((num_errors + 1))
+      fi
+    fi
+    if [ $((num_solutions)) -eq 0 ]; then
+        avg_ms_per_solution=0
+    else
+        avg_ms_per_solution=$((total_ms / num_solutions))
+    fi
+    echo -ne "#instances: $((num_solutions + num_invalid + num_errors)), #solutions: $num_solutions, #invalid: $num_invalid, #errors: $num_errors, average ms/solution: $avg_ms_per_solution\r"
+done
+echo
+echo "----------------------------------------------------------------------"
+echo "To re-run this test, run the following commands:"
+echo "    git clone https://github.com/tig-foundation/tig-monorepo.git"
+echo "    cd tig-monorepo"
+echo "    git checkout origin/$CHALLENGE/$ALGORITHM"
+echo "    bash scripts/test_algorithm_performance.sh"
+echo "----------------------------------------------------------------------"
+echo "Share your results on https://www.reddit.com/r/TheInnovationGame"
+echo "----------------------------------------------------------------------"
--- a/scripts/test_algorithm_performance.sh
+++ b/scripts/test_algorithm_performance.sh
@ -1,86 +0,0 @@
-#!/bin/bash
-set -e
-
-echo "Enter settings (JSON string):"
-read settings
-
-echo "Enter starting nonce:"
-read start_nonce
-
-echo "Enter number of nonces:"
-read num_nonces
-
-challenge_id=$(echo "$settings" | jq -r '.challenge_id')
-algorithm_id=$(echo "$settings" | jq -r '.algorithm_id')
-
-if [[ -z "$challenge_id" || "$challenge_id" == "null" ]]; then
-  echo "Error: Unable to parse challenge_id"
-  exit 1
-fi
-if [[ -z "$algorithm_id" || "$algorithm_id" == "null" ]]; then
-  echo "Error: Unable to parse algorithm_id"
-  exit 1
-fi
-
-echo "Fetching names for challenge '$challenge_id' and algorithm '$algorithm_id'"
-block_id=$(curl -s https://mainnet-api.tig.foundation/get-block | jq -r '.block.id')
-algorithm_name=$(curl -s "https://mainnet-api.tig.foundation/get-algorithms?block_id=$block_id" | jq -r --arg ID "$algorithm_id" '.algorithms[] | select(.id == $ID) | .details.name')
-challenge_name=$(curl -s "https://mainnet-api.tig.foundation/get-challenges?block_id=$block_id" | jq -r --arg ID "$challenge_id" '.challenges[] | select(.id == $ID) | .details.name')
-if [[ -z "$algorithm_name" || "$algorithm_name" == "null" ]]; then
-  echo "Error: Unable to find algorithm_name for '$algorithm_id'"
-  exit 1
-fi
-if [[ -z "$challenge_name" || "$challenge_name" == "null" ]]; then
-  echo "Error: Unable to find challenge_name for '$challenge_id'"
-  exit 1
-fi
-
-branch="test_performance/${challenge_name}/${algorithm_name}"
-if git show-ref --quiet $branch; then
-    echo "Branch $branch already exists. Switching"
-    git checkout $branch
-else 
-    echo "Create new branch with name $branch? (y/n)"
-    read confirm
-    if [ "$confirm" != "y" ]; then
-        echo "Aborting."
-        exit 1
-    fi
-    git fetch origin
-    git checkout -b $branch origin/blank_slate
-fi
-git pull origin $challenge_name/$algorithm_name --no-edit
-
-cargo build -p tig-worker --release --features ${challenge_name}_${algorithm_name}
-
-solutions=0
-invalid=0
-total_ms=0
-
-echo "------------------------------------------------------------"
-echo "Testing performance of $challenge_name/$algorithm_name"
-echo "Settings: $settings"
-echo "Starting nonce: $start_nonce"
-echo "Number of nonces: $num_nonces"
-echo -ne ""
-for ((nonce=start_nonce; nonce<start_nonce+num_nonces; nonce++)); do
-    start_time=$(date +%s%3N)
-    ./target/release/tig-worker compute_solution "$settings" $nonce ./tig-algorithms/wasm/$challenge_name/$algorithm_name.wasm > /dev/null 2>&1
-    exit_code=$?
-    end_time=$(date +%s%3N)
-    duration=$((end_time - start_time))
-    total_ms=$((total_ms + duration))
-    if [ $exit_code -eq 0 ]; then
-        solutions=$((solutions + 1))
-    else
-        invalid=$((invalid + 1))
-    fi
-    if [ $((solutions)) -eq 0 ]; then
-        avg_ms_per_solution=0
-    else
-        avg_ms_per_solution=$((total_ms / solutions))
-    fi
-    echo -ne "#instances: $((solutions + invalid)), #solutions: $solutions, #invalid: $invalid, average ms/solution: $avg_ms_per_solution, average ms/nonce: $((total_ms / (solutions + invalid)))\r"
-done
-echo
-echo "------------------------------------------------------------"
--- a/scripts/verify_benchmark_solutions.sh
+++ b/scripts/verify_benchmark_solutions.sh
@ -1,9 +1,12 @@
 #!/bin/bash
 set -e

-if [ ! -f ./target/release/tig-worker ]; then
+REPO_DIR=$(dirname $(dirname "$(realpath "$0")"))
+TIG_WORKER_PATH="$REPO_DIR/target/release/tig-worker"
+
+if [ ! -f $TIG_WORKER_PATH ]; then
    echo "Error: tig-worker binary not found at ./target/release/tig-worker"
-    echo "Run: cargo build -p tig-worker --release"
+    echo "Run: cd $REPO_DIR && cargo build -p tig-worker --release"
    exit 1
 fi

@ -64,8 +67,6 @@ for i in $(seq 0 $(($solutions_count - 1))); do
        echo "Ok"
    else
        echo "Mismatch. Actual: $compute_output"
-        rm $algorithm_id.wasm
-        exit 1
    fi
 done
 rm $algorithm_id.wasm
--- a/tig-worker/README.md
+++ b/tig-worker/README.md
@ -62,6 +62,28 @@ Options:
  -h, --help           Print help
 ```

+**Example:**
+```
+SETTINGS='{"challenge_id":"c001","difficulty":[50,300],"algorithm_id":"","player_id":"","block_id":""}'
+NONCE=0
+WASM=./tig-algorithms/wasm/satisfiability/sprint_sat.wasm
+./target/release/tig-worker compute_solution $SETTINGS $NONCE $WASM 
+```
+
+**Notes**:
+* `challenge_id` must be set:
+  * `c001` is satisfiability
+  * `c002` is vehicle_routing
+  * `c003` is knapsack
+  * `c004` is vector_search
+* Recommended low difficulties for testing are:
+  * satisfiability [50,300]
+  * vehicle_routing [40, 250]
+  * knapsack [50, 10]
+  * vector_search [10, 350]
+* You can query the latest difficulties by using the `bash scripts/list_challenges.sh`
+* You can test the performance of an algorithm using `bash scripts/test_algorithm.sh`
+
 ## Verify Solution

 Given settings, nonce and a solution, `tig-worker` verifies the solution is a valid solution for the challenge instance.
@ -82,6 +104,19 @@ Options:
  -h, --help  Print help
 ```

+**Example:**
+```
+SETTINGS='{"challenge_id":"c001","difficulty":[50,300],"algorithm_id":"","player_id":"","block_id":""}'
+NONCE=0
+SOLUTION='{"variables":[1,0,1,1,1,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,1,1,0,0,1,1,0,0,1,0,1,0,1,1,0,0,0,1,1,0,1,1,0]}'
+./target/release/tig-worker verify_solution $SETTINGS $NONCE $SOLUTION
+```
+
+**Notes**:
+* You can list all active benchmark ids with `scripts/list_benchmark_ids.sh`
+* You get benchmark data with  `scripts/list_benchmark_ids.sh`
+* You verify a benchmark's solutions, runtime_signature and fuel_consumed with  `scripts/verify_benchmark.sh`
+
 # License

 [End User License Agreement](../docs/agreements/end_user_license_agreement.pdf)