rommapp/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-02-18 00:27:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Run worker as a native process

Browse Source
This commit is contained in:
Georges-Antoine Assi
2025-08-06 21:57:32 -04:00
parent ebe271df34
commit 6b307afcd3
7 changed files with 58 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.vscode/tasks.json vendored
View File

@@ -14,12 +14,6 @@
"command": "cd backend && uv run python3 main.py",
"problemMatcher": []
},
{
"label": "Launch worker",
"type": "shell",
"command": "cd backend && uv run python3 worker.py",
"problemMatcher": []
},
{
"label": "Execute tests",
"type": "shell",

7
DEVELOPER_SETUP.md
View File

@@ -117,13 +117,6 @@ cd backend
uv run python3 main.py
```
#### - Start a worker
```sh
cd backend
uv run python3 worker.py
```
### Setting up the frontend
#### - Install node.js dependencies

2
Dockerfile
View File

@@ -74,6 +74,8 @@ COPY pyproject.toml uv.lock* .python-version /app/
# Install Python dependencies
RUN uv sync --all-extras
ENV PATH="/app/.venv/bin:${PATH}"
# Copy entrypoint script
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh

20
backend/main.py
View File

@@ -11,6 +11,9 @@ from config import (
DEV_HOST,
DEV_PORT,
DISABLE_CSRF_PROTECTION,
ENABLE_SCHEDULED_RESCAN,
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
IS_PYTEST_RUN,
ROMM_AUTH_SECRET_KEY,
SENTRY_DSN,
@@ -41,7 +44,11 @@ from handler.auth.hybrid_auth import HybridAuthBackend
from handler.auth.middleware import CustomCSRFMiddleware, SessionMiddleware
from handler.socket_handler import socket_handler
from logger.log_middleware import LOGGING_CONFIG, CustomLoggingMiddleware
from logger.logger import log
from starlette.middleware.authentication import AuthenticationMiddleware
from tasks.scheduled.scan_library import scan_library_task
from tasks.scheduled.update_launchbox_metadata import update_launchbox_metadata_task
from tasks.scheduled.update_switch_titledb import update_switch_titledb_task
from utils import get_version
from utils.context import (
ctx_aiohttp_session,
@@ -139,3 +146,16 @@ if __name__ == "__main__":
# Run application
app.add_middleware(CustomLoggingMiddleware)
uvicorn.run("main:app", host=DEV_HOST, port=DEV_PORT, reload=True, access_log=False)
# Initialize scheduled tasks
if ENABLE_SCHEDULED_RESCAN:
log.info("Starting scheduled rescan")
scan_library_task.init()
if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:
log.info("Starting scheduled update switch titledb")
update_switch_titledb_task.init()
if ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA:
log.info("Starting scheduled update launchbox metadata")
update_launchbox_metadata_task.init()

40
backend/worker.py
View File

@@ -1,40 +0,0 @@
import sentry_sdk
from config import (
ENABLE_SCHEDULED_RESCAN,
ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA,
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
SENTRY_DSN,
)
from handler.redis_handler import redis_client
from logger.logger import log, unify_logger
from rq import Queue, Worker
from tasks.scheduled.scan_library import scan_library_task
from tasks.scheduled.update_launchbox_metadata import update_launchbox_metadata_task
from tasks.scheduled.update_switch_titledb import update_switch_titledb_task
from utils import get_version
unify_logger("rq.worker")
listen = ("high", "default", "low")
sentry_sdk.init(
dsn=SENTRY_DSN,
release=f"romm@{get_version()}",
)
if __name__ == "__main__":
# Initialize scheduled tasks
if ENABLE_SCHEDULED_RESCAN:
log.info("Starting scheduled rescan")
scan_library_task.init()
if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:
log.info("Starting scheduled update switch titledb")
update_switch_titledb_task.init()
if ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA:
log.info("Starting scheduled update launchbox metadata")
update_launchbox_metadata_task.init()
worker = Worker([Queue(name, connection=redis_client) for name in listen])
worker.work()

23
docker/init_scripts/init
View File

@@ -162,7 +162,7 @@ start_bin_valkey-server() {
}
# Commands to start RQ scheduler
start_bin_scheduler() {
start_bin_rq_scheduler() {
info_log "Starting RQ scheduler"
RQ_REDIS_HOST=${REDIS_HOST:-127.0.0.1} \
@@ -173,7 +173,18 @@ start_bin_scheduler() {
RQ_REDIS_SSL=${REDIS_SSL:-0} \
rqscheduler \
--path /backend \
--pid /tmp/scheduler.pid &
--pid /tmp/rq_scheduler.pid &
}
# Commands to start RQ worker
start_bin_rq_worker() {
info_log "Starting rq worker"
rq worker \
--path /backend \
--pid /tmp/rq_worker.pid \
--url "redis${REDIS_SSL:+s}://${REDIS_USERNAME:-""}:${REDIS_PASSWORD:-""}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" \
high default low &
}
# function that runs our independent python scripts and creates corresponding PID files,
@@ -223,8 +234,8 @@ stop_process_pid() {
shutdown() {
# shutdown in reverse order
stop_process_pid worker
stop_process_pid scheduler
stop_process_pid rq_worker
stop_process_pid rq_scheduler
stop_process_pid watcher
stop_process_pid nginx
stop_process_pid gunicorn
@@ -264,10 +275,10 @@ while ! ((exited)); do
# only start the scheduler if enabled
if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" || ${ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA} == "true" ]]; then
watchdog_process_pid bin scheduler
watchdog_process_pid bin rq_scheduler
fi
watchdog_process_pid python worker
watchdog_process_pid bin rq_worker
# only start the watcher if enabled
if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then

20
entrypoint.sh
View File

@@ -32,9 +32,27 @@ function handle_termination() {
trap handle_termination SIGTERM SIGINT
# Start all services in the background
echo "Starting backend..."
cd /app/backend
uv run python main.py &
OBJC_DISABLE_INITIALIZE_FORK_SAFETY=1 uv run python worker.py &
echo "Starting RQ scheduler..."
RQ_REDIS_HOST=${REDIS_HOST:-127.0.0.1} \
RQ_REDIS_PORT=${REDIS_PORT:-6379} \
RQ_REDIS_USERNAME=${REDIS_USERNAME:-""} \
RQ_REDIS_PASSWORD=${REDIS_PASSWORD:-""} \
RQ_REDIS_DB=${REDIS_DB:-0} \
RQ_REDIS_SSL=${REDIS_SSL:-0} \
rqscheduler \
--path /backend \
--pid /tmp/rq_scheduler.pid &
echo "Starting RQ worker..."
rq worker \
--path /backend \
--pid /tmp/rq_worker.pid \
--url "redis${REDIS_SSL:+s}://${REDIS_USERNAME:-""}:${REDIS_PASSWORD:-""}@${REDIS_HOST:-127.0.0.1}:${REDIS_PORT:-6379}/${REDIS_DB:-0}" \
high default low &
# Start the frontend dev server
cd /app/frontend