From 6b307afcd3a7a06a490956fb6269d7e8b3497eac Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Wed, 6 Aug 2025 21:57:32 -0400 Subject: [PATCH] Run worker as a native process --- .vscode/tasks.json | 6 ------ DEVELOPER_SETUP.md | 7 ------- Dockerfile | 2 ++ backend/main.py | 20 ++++++++++++++++++++ backend/worker.py | 40 ---------------------------------------- docker/init_scripts/init | 23 +++++++++++++++++------ entrypoint.sh | 20 +++++++++++++++++++- 7 files changed, 58 insertions(+), 60 deletions(-) delete mode 100644 backend/worker.py diff --git a/.vscode/tasks.json b/.vscode/tasks.json index f7eb9700f..03bfd165b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -14,12 +14,6 @@ "command": "cd backend && uv run python3 main.py", "problemMatcher": [] }, - { - "label": "Launch worker", - "type": "shell", - "command": "cd backend && uv run python3 worker.py", - "problemMatcher": [] - }, { "label": "Execute tests", "type": "shell", diff --git a/DEVELOPER_SETUP.md b/DEVELOPER_SETUP.md index 6b0abb8cf..4f5d2a5f3 100644 --- a/DEVELOPER_SETUP.md +++ b/DEVELOPER_SETUP.md @@ -117,13 +117,6 @@ cd backend uv run python3 main.py ``` -#### - Start a worker - -```sh -cd backend -uv run python3 worker.py -``` - ### Setting up the frontend #### - Install node.js dependencies diff --git a/Dockerfile b/Dockerfile index d9da9a2dc..865acf973 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,6 +74,8 @@ COPY pyproject.toml uv.lock* .python-version /app/ # Install Python dependencies RUN uv sync --all-extras +ENV PATH="/app/.venv/bin:${PATH}" + # Copy entrypoint script COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/backend/main.py b/backend/main.py index 597e80e8d..a9711d793 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,6 +11,9 @@ from config import ( DEV_HOST, DEV_PORT, DISABLE_CSRF_PROTECTION, + ENABLE_SCHEDULED_RESCAN, + ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, + ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, IS_PYTEST_RUN, ROMM_AUTH_SECRET_KEY, SENTRY_DSN, @@ -41,7 +44,11 @@ from handler.auth.hybrid_auth import HybridAuthBackend from handler.auth.middleware import CustomCSRFMiddleware, SessionMiddleware from handler.socket_handler import socket_handler from logger.log_middleware import LOGGING_CONFIG, CustomLoggingMiddleware +from logger.logger import log from starlette.middleware.authentication import AuthenticationMiddleware +from tasks.scheduled.scan_library import scan_library_task +from tasks.scheduled.update_launchbox_metadata import update_launchbox_metadata_task +from tasks.scheduled.update_switch_titledb import update_switch_titledb_task from utils import get_version from utils.context import ( ctx_aiohttp_session, @@ -139,3 +146,16 @@ if __name__ == "__main__": # Run application app.add_middleware(CustomLoggingMiddleware) uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=True, access_log=False) + + # Initialize scheduled tasks + if ENABLE_SCHEDULED_RESCAN: + log.info("Starting scheduled rescan") + scan_library_task.init() + + if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: + log.info("Starting scheduled update switch titledb") + update_switch_titledb_task.init() + + if ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA: + log.info("Starting scheduled update launchbox metadata") + update_launchbox_metadata_task.init() diff --git a/backend/worker.py b/backend/worker.py deleted file mode 100644 index 428735124..000000000 --- a/backend/worker.py +++ /dev/null @@ -1,40 +0,0 @@ -import sentry_sdk -from config import ( - ENABLE_SCHEDULED_RESCAN, - ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, - ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, - SENTRY_DSN, -) -from handler.redis_handler import redis_client -from logger.logger import log, unify_logger -from rq import Queue, Worker -from tasks.scheduled.scan_library import scan_library_task -from tasks.scheduled.update_launchbox_metadata import update_launchbox_metadata_task -from tasks.scheduled.update_switch_titledb import update_switch_titledb_task -from utils import get_version - -unify_logger("rq.worker") - -listen = ("high", "default", "low") - -sentry_sdk.init( - dsn=SENTRY_DSN, - release=f"romm@{get_version()}", -) - -if __name__ == "__main__": - # Initialize scheduled tasks - if ENABLE_SCHEDULED_RESCAN: - log.info("Starting scheduled rescan") - scan_library_task.init() - - if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: - log.info("Starting scheduled update switch titledb") - update_switch_titledb_task.init() - - if ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA: - log.info("Starting scheduled update launchbox metadata") - update_launchbox_metadata_task.init() - - worker = Worker([Queue(name, connection=redis_client) for name in listen]) - worker.work() diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 78c24bfe3..dfe1f4f3e 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -162,7 +162,7 @@ start_bin_valkey-server() { } # Commands to start RQ scheduler -start_bin_scheduler() { +start_bin_rq_scheduler() { info_log "Starting RQ scheduler" RQ_REDIS_HOST=${REDIS_HOST:-127.0.0.1} \ @@ -173,7 +173,18 @@ start_bin_scheduler() { RQ_REDIS_SSL=${REDIS_SSL:-0} \ rqscheduler \ --path /backend \ - --pid /tmp/scheduler.pid & + --pid /tmp/rq_scheduler.pid & +} + +# Commands to start RQ worker +start_bin_rq_worker() { + info_log "Starting rq worker" + + rq worker \ + --path /backend \ + --pid /tmp/rq_worker.pid \ + --url "redis${REDIS_SSL:+s}://${REDIS_USERNAME:-""}:${REDIS_PASSWORD:-""}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" \ + high default low & } # function that runs our independent python scripts and creates corresponding PID files, @@ -223,8 +234,8 @@ stop_process_pid() { shutdown() { # shutdown in reverse order - stop_process_pid worker - stop_process_pid scheduler + stop_process_pid rq_worker + stop_process_pid rq_scheduler stop_process_pid watcher stop_process_pid nginx stop_process_pid gunicorn @@ -264,10 +275,10 @@ while ! ((exited)); do # only start the scheduler if enabled if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" || ${ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA} == "true" ]]; then - watchdog_process_pid bin scheduler + watchdog_process_pid bin rq_scheduler fi - watchdog_process_pid python worker + watchdog_process_pid bin rq_worker # only start the watcher if enabled if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then diff --git a/entrypoint.sh b/entrypoint.sh index 00308fcc4..87590a5bc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -32,9 +32,27 @@ function handle_termination() { trap handle_termination SIGTERM SIGINT # Start all services in the background +echo "Starting backend..." cd /app/backend uv run python main.py & -OBJC_DISABLE_INITIALIZE_FORK_SAFETY=1 uv run python worker.py & + +echo "Starting RQ scheduler..." +RQ_REDIS_HOST=${REDIS_HOST:-127.0.0.1} \ + RQ_REDIS_PORT=${REDIS_PORT:-6379} \ + RQ_REDIS_USERNAME=${REDIS_USERNAME:-""} \ + RQ_REDIS_PASSWORD=${REDIS_PASSWORD:-""} \ + RQ_REDIS_DB=${REDIS_DB:-0} \ + RQ_REDIS_SSL=${REDIS_SSL:-0} \ + rqscheduler \ + --path /backend \ + --pid /tmp/rq_scheduler.pid & + +echo "Starting RQ worker..." +rq worker \ + --path /backend \ + --pid /tmp/rq_worker.pid \ + --url "redis${REDIS_SSL:+s}://${REDIS_USERNAME:-""}:${REDIS_PASSWORD:-""}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" \ + high default low & # Start the frontend dev server cd /app/frontend