diff --git a/tig-benchmarker/.env b/tig-benchmarker/.env index c365bc4..19a17ee 100644 --- a/tig-benchmarker/.env +++ b/tig-benchmarker/.env @@ -1,5 +1,7 @@ +# Version of all benchmarker containers +VERSION=0.0.1 # Set to 1 to enable verbose logging -VERBOSE= +VERBOSE=1 POSTGRES_USER=postgres POSTGRES_PASSWORD=mysecretpassword @@ -7,15 +9,18 @@ POSTGRES_DB=postgres UI_PORT=80 DB_PORT=5432 +# This is used by both master and slave MASTER_PORT=5115 -# Defaults to 0.0.0.0 -MASTER_IP= +# This is used by slave to connect to master. Set to 172.17.0.1 if master and slave are running on same server +MASTER_IP=172.17.0.1 -# Directory for slave to download algorithms. Defaults to ./lib -ALGORITHMS_DIR= -# Directory for slave to store results. Defaults to ./results -RESULTS_DIR= -# Seconds for results to live. Defaults to 300s -TTL= +# Path to config file for slave. Mounts to /app/config.json inside slave container +SLAVE_CONFIG=./slave/config.json +# Directory for slave to download algorithms. Mounts to /app/algorithms inside slave containers +ALGORITHMS_DIR=./algorithms +# Directory for slave to store results. Mounts to /app/results inside slave containers +RESULTS_DIR=./results +# Seconds for results to live +TTL=300 # Name of the slave. Defaults to randomly generated name SLAVE_NAME= \ No newline at end of file diff --git a/tig-benchmarker/docker-compose-master.yml b/tig-benchmarker/docker-compose-master.yml index 74451ad..0ba14ec 100644 --- a/tig-benchmarker/docker-compose-master.yml +++ b/tig-benchmarker/docker-compose-master.yml @@ -1,8 +1,6 @@ services: db: - build: - context: ./postgres - dockerfile: Dockerfile + image: ghcr.io/tig-foundation/tig-monorepo/benchmarker/postgres:${VERSION} container_name: db restart: unless-stopped environment: @@ -22,9 +20,7 @@ services: retries: 5 master: - build: - context: ./ - dockerfile: ./master/Dockerfile + image: ghcr.io/tig-foundation/tig-monorepo/benchmarker/master:${VERSION} environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -42,9 +38,7 @@ services: condition: service_healthy ui: - build: - context: ./ui - dockerfile: Dockerfile + image: ghcr.io/tig-foundation/tig-monorepo/benchmarker/ui:${VERSION} restart: unless-stopped networks: - tig-benchmarker @@ -54,9 +48,7 @@ services: - master nginx: - build: - context: ./nginx - dockerfile: Dockerfile + image: ghcr.io/tig-foundation/tig-monorepo/benchmarker/nginx:${VERSION} ports: - "${UI_PORT}:80" networks: diff --git a/tig-benchmarker/docker-compose-slave.yml b/tig-benchmarker/docker-compose-slave.yml index 98cfb0e..237c7b1 100644 --- a/tig-benchmarker/docker-compose-slave.yml +++ b/tig-benchmarker/docker-compose-slave.yml @@ -2,7 +2,8 @@ version: "3.8" x-common: &common volumes: - - ./:/app + - ${ALGORITHMS_DIR}:/app/algorithms + - ${RESULTS_DIR}:/app/results command: ["sleep", "infinity"] x-common-gpu: &common-gpu @@ -16,43 +17,40 @@ x-common-gpu: &common-gpu services: slave: - build: - context: ./ - dockerfile: ./slave/Dockerfile + image: ghcr.io/tig-foundation/tig-monorepo/benchmarker/slave:${VERSION} volumes: - /var/run/docker.sock:/var/run/docker.sock - - ./:/app + - ${ALGORITHMS_DIR}:/app/algorithms + - ${RESULTS_DIR}:/app/results + - ${SLAVE_CONFIG}:/app/config.json environment: - VERBOSE=${VERBOSE} - SLAVE_NAME=${SLAVE_NAME} - MASTER_PORT=${MASTER_PORT} - MASTER_IP=${MASTER_IP} - - ALGORITHMS_DIR=${ALGORITHMS_DIR} - - RESULTS_DIR=${RESULTS_DIR} - TTL=${TTL} - command: ["python", "slave/main.py", "slave/config.json"] satisfiability: <<: *common - image: ghcr.io/tig-foundation/tig-monorepo/satisfiability/runtime:0.0.1 + image: ghcr.io/tig-foundation/tig-monorepo/satisfiability/runtime:${VERSION} container_name: satisfiability vehicle_routing: <<: *common - image: ghcr.io/tig-foundation/tig-monorepo/vehicle_routing/runtime:0.0.1 + image: ghcr.io/tig-foundation/tig-monorepo/vehicle_routing/runtime:${VERSION} container_name: vehicle_routing knapsack: <<: *common - image: ghcr.io/tig-foundation/tig-monorepo/knapsack/runtime:0.0.1 + image: ghcr.io/tig-foundation/tig-monorepo/knapsack/runtime:${VERSION} container_name: knapsack vector_search: <<: [*common, *common-gpu] - image: ghcr.io/tig-foundation/tig-monorepo/vector_search/runtime:0.0.1 + image: ghcr.io/tig-foundation/tig-monorepo/vector_search/runtime:${VERSION} container_name: vector_search hypergraph: <<: [*common, *common-gpu] - image: ghcr.io/tig-foundation/tig-monorepo/hypergraph/runtime:0.0.1 + image: ghcr.io/tig-foundation/tig-monorepo/hypergraph/runtime:${VERSION} container_name: hypergraph \ No newline at end of file diff --git a/tig-benchmarker/master/main.py b/tig-benchmarker/master/main.py index f4189c9..81eeb6c 100644 --- a/tig-benchmarker/master/main.py +++ b/tig-benchmarker/master/main.py @@ -1,8 +1,5 @@ -import argparse -import json import logging import os -import threading import time from master.data_fetcher import * from master.difficulty_sampler import * @@ -53,10 +50,6 @@ def main(): time.sleep(5) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="TIG Benchmarker") - - args = parser.parse_args() - logging.basicConfig( format='%(levelname)s - [%(name)s] - %(message)s', level=logging.DEBUG if os.environ.get("VERBOSE") else logging.INFO diff --git a/tig-benchmarker/nginx/Dockerfile b/tig-benchmarker/nginx/Dockerfile index 9ded20d..16181bb 100644 --- a/tig-benchmarker/nginx/Dockerfile +++ b/tig-benchmarker/nginx/Dockerfile @@ -1,3 +1,3 @@ FROM nginx:alpine -COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file +COPY nginx/nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/tig-benchmarker/postgres/Dockerfile b/tig-benchmarker/postgres/Dockerfile index 69be141..3b37e81 100644 --- a/tig-benchmarker/postgres/Dockerfile +++ b/tig-benchmarker/postgres/Dockerfile @@ -1,3 +1,3 @@ FROM postgres:17 -COPY init.sql /docker-entrypoint-initdb.d/ \ No newline at end of file +COPY postgres/init.sql /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/tig-benchmarker/slave/Dockerfile b/tig-benchmarker/slave/Dockerfile index 06665ce..1b20289 100644 --- a/tig-benchmarker/slave/Dockerfile +++ b/tig-benchmarker/slave/Dockerfile @@ -1,14 +1,15 @@ -FROM ubuntu:24.04 +FROM python:3.13-slim WORKDIR /app -RUN apt update && apt install -y curl python3 python3-pip COPY slave/requirements.txt requirements.txt -RUN pip3 install -r requirements.txt --break-system-packages --no-cache-dir +RUN pip3 install -r requirements.txt --no-cache-dir -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl gnupg lsb-release \ - && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list \ - && apt-get update && apt-get install -y docker-ce \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get update && apt-get install -y docker.io \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +COPY common common +COPY slave/main.py main.py +COPY slave/config.json config.json + +CMD ["python", "main.py"] \ No newline at end of file diff --git a/tig-benchmarker/slave/main.py b/tig-benchmarker/slave/main.py index bcee803..9b71dd7 100644 --- a/tig-benchmarker/slave/main.py +++ b/tig-benchmarker/slave/main.py @@ -274,7 +274,7 @@ def process_batch(pool, algorithms_dir, config, results_dir): try: batch_id = PENDING_BATCH_IDS.pop() except KeyError: - logger.debug("No pending batches") + logger.debug("no pending batches") time.sleep(1) return @@ -401,7 +401,8 @@ def wrap_thread(func, *args): time.sleep(5) -def main(config_path: str): +def main(): + config_path = "config.json" if not os.path.exists(config_path): logger.error(f"Config file not found at path: {config_path}") sys.exit(1) @@ -419,8 +420,8 @@ def main(config_path: str): sys.exit(1) master_port = int(master_port) - algorithms_dir = os.getenv("ALGORITHMS_DIR") or "lib" - results_dir = os.getenv("RESULTS_DIR") or "results" + algorithms_dir = "algorithms" + results_dir = "results" ttl = int(os.getenv("TTL") or 300) print(f"Starting slave with config:") @@ -462,15 +463,10 @@ def main(config_path: str): wrap_thread(poll_batches, headers, master_ip, master_port, results_dir) -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="TIG Slave Benchmarker") - parser.add_argument("config", type=str, help="Path to config file") - - args = parser.parse_args() - +if __name__ == "__main__": logging.basicConfig( format='%(levelname)s - [%(name)s] - %(message)s', level=logging.DEBUG if os.getenv("VERBOSE") else logging.INFO ) - main(args.config) + main() diff --git a/tig-benchmarker/ui/Dockerfile b/tig-benchmarker/ui/Dockerfile index a3fbb68..cddebb7 100644 --- a/tig-benchmarker/ui/Dockerfile +++ b/tig-benchmarker/ui/Dockerfile @@ -2,7 +2,7 @@ FROM node:latest AS build WORKDIR /app -COPY . . +COPY ui/ . RUN npm install -g @angular/cli RUN npm install --legacy-peer-deps @@ -14,7 +14,7 @@ RUN ng build FROM nginx:latest COPY --from=build /app/dist/tig-brenchmarker-ui/browser /usr/share/nginx/html -COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY ui/nginx.conf /etc/nginx/conf.d/default.conf EXPOSE 80