From a31a8504c230b43d646aef48452ffb29dd35b824 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Sun, 31 Aug 2025 01:39:57 -0300 Subject: [PATCH] feat: Add scheduled task to sync RetroAchievements progress Add a new scheduled task that syncs RetroAchievements progress for all users with a RetroAchievements username. Environment variables: - `ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC`: Enable or disable the task (default: `false`) - `SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON`: Cron string to schedule the task (default: "0 4 * * *" - daily at 4 AM) --- backend/config/__init__.py | 7 ++ backend/handler/database/users_handler.py | 61 ++++++++++-- backend/pytest.ini | 1 - backend/startup.py | 7 ++ .../sync_retroachievements_progress.py | 61 ++++++++++++ .../test_sync_retroachievements_progress.py | 93 +++++++++++++++++++ env.template | 2 + 7 files changed, 221 insertions(+), 11 deletions(-) create mode 100644 backend/tasks/scheduled/sync_retroachievements_progress.py create mode 100644 backend/tests/tasks/test_sync_retroachievements_progress.py diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 12a43258e..7092cf908 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -158,6 +158,13 @@ SCHEDULED_CONVERT_IMAGES_TO_WEBP_CRON: Final = os.environ.get( "SCHEDULED_CONVERT_IMAGES_TO_WEBP_CRON", "0 4 * * *", # At 4:00 AM every day ) +ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC: Final[bool] = str_to_bool( + os.environ.get("ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC", "false") +) +SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON: Final[str] = os.environ.get( + "SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON", + "0 4 * * *", # At 4:00 AM every day +) # EMULATION DISABLE_EMULATOR_JS = str_to_bool(os.environ.get("DISABLE_EMULATOR_JS", "false")) diff --git a/backend/handler/database/users_handler.py b/backend/handler/database/users_handler.py index c605f3690..99049a8e2 100644 --- a/backend/handler/database/users_handler.py +++ b/backend/handler/database/users_handler.py @@ -2,13 +2,40 @@ from collections.abc import Sequence from decorators.database import begin_session from models.user import Role, User -from sqlalchemy import delete, func, select, update +from sqlalchemy import and_, delete, func, not_, select, update from sqlalchemy.orm import Session +from sqlalchemy.sql import Delete, Select, Update from .base_handler import DBBaseHandler class DBUsersHandler(DBBaseHandler): + def filter[QueryT: Select[tuple[User]] | Update | Delete]( + self, + query: QueryT, + *, + usernames: Sequence[str] = (), + emails: Sequence[str] = (), + roles: Sequence[Role] = (), + has_ra_username: bool | None = None, + ) -> QueryT: + if usernames: + query = query.filter( + func.lower(User.username).in_([u.lower() for u in usernames]) + ) + if emails: + query = query.filter( + func.lower(User.email).in_([e.lower() for e in emails]) + ) + if roles: + query = query.filter(User.role.in_(roles)) + if has_ra_username is not None: + predicate = and_(User.ra_username != "", User.ra_username.isnot(None)) + if not has_ra_username: + predicate = not_(predicate) + query = query.filter(predicate) + return query + @begin_session def add_user(self, user: User, session: Session = None) -> User: return session.merge(user) @@ -17,15 +44,13 @@ class DBUsersHandler(DBBaseHandler): def get_user_by_username( self, username: str, session: Session = None ) -> User | None: - return session.scalar( - select(User).filter(func.lower(User.username) == username.lower()).limit(1) - ) + query = self.filter(select(User), usernames=[username]) + return session.scalar(query.limit(1)) @begin_session def get_user_by_email(self, email: str, session: Session = None) -> User | None: - return session.scalar( - select(User).filter(func.lower(User.email) == email.lower()).limit(1) - ) + query = self.filter(select(User), emails=[email]) + return session.scalar(query.limit(1)) @begin_session def get_user(self, id: int, session: Session = None) -> User | None: @@ -42,8 +67,23 @@ class DBUsersHandler(DBBaseHandler): return session.query(User).filter_by(id=id).one() @begin_session - def get_users(self, session: Session = None) -> Sequence[User]: - return session.scalars(select(User)).all() + def get_users( + self, + *, + usernames: Sequence[str] = (), + emails: Sequence[str] = (), + roles: Sequence[Role] = (), + has_ra_username: bool | None = None, + session: Session = None, + ) -> Sequence[User]: + query = self.filter( + select(User), + usernames=usernames, + emails=emails, + roles=roles, + has_ra_username=has_ra_username, + ) + return session.scalars(query).all() @begin_session def delete_user(self, id: int, session: Session = None): @@ -55,4 +95,5 @@ class DBUsersHandler(DBBaseHandler): @begin_session def get_admin_users(self, session: Session = None) -> Sequence[User]: - return session.scalars(select(User).filter_by(role=Role.ADMIN)).all() + query = self.filter(select(User), roles=[Role.ADMIN]) + return session.scalars(query).all() diff --git a/backend/pytest.ini b/backend/pytest.ini index bf47b9d19..d9d4ccb46 100644 --- a/backend/pytest.ini +++ b/backend/pytest.ini @@ -9,7 +9,6 @@ env = ROMM_DB_DRIVER=mariadb IGDB_CLIENT_ID=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx IGDB_CLIENT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - RETROACHIEVEMENTS_USERNAME=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx RETROACHIEVEMENTS_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx MOBYGAMES_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx SCREENSCRAPER_USER=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/backend/startup.py b/backend/startup.py index 6b2b68655..e55a69ad1 100644 --- a/backend/startup.py +++ b/backend/startup.py @@ -6,6 +6,7 @@ import sentry_sdk from config import ( ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP, ENABLE_SCHEDULED_RESCAN, + ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC, ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SENTRY_DSN, @@ -24,6 +25,9 @@ from models.firmware import FIRMWARE_FIXTURES_DIR, KNOWN_BIOS_KEY from opentelemetry import trace from tasks.scheduled.convert_images_to_webp import convert_images_to_webp_task from tasks.scheduled.scan_library import scan_library_task +from tasks.scheduled.sync_retroachievements_progress import ( + sync_retroachievements_progress_task, +) from tasks.scheduled.update_launchbox_metadata import update_launchbox_metadata_task from tasks.scheduled.update_switch_titledb import update_switch_titledb_task from utils import get_version @@ -53,6 +57,9 @@ async def main() -> None: if ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP: log.info("Starting scheduled convert images to webp") convert_images_to_webp_task.init() + if ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC: + log.info("Starting scheduled RetroAchievements progress sync") + sync_retroachievements_progress_task.init() log.info("Initializing cache with fixtures data") await conditionally_set_cache( diff --git a/backend/tasks/scheduled/sync_retroachievements_progress.py b/backend/tasks/scheduled/sync_retroachievements_progress.py new file mode 100644 index 000000000..6a260fa1f --- /dev/null +++ b/backend/tasks/scheduled/sync_retroachievements_progress.py @@ -0,0 +1,61 @@ +from typing import cast + +from config import ( + ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC, + SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON, +) +from handler.database import db_user_handler +from handler.metadata import meta_ra_handler +from handler.metadata.ra_handler import RA_API_ENABLED, RAUserProgression +from logger.logger import log +from tasks.tasks import PeriodicTask +from utils.context import initialize_context + + +class SyncRetroAchievementsProgressTask(PeriodicTask): + def __init__(self): + super().__init__( + title="Scheduled RetroAchievements progress sync", + description="Updates RetroAchievements progress for all users", + enabled=ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC, + cron_string=SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON, + manual_run=False, + func="tasks.scheduled.sync_retroachievements_progress.sync_retroachievements_progress_task.run", + ) + + @initialize_context() + async def run(self) -> None: + if not RA_API_ENABLED: + log.warning("RetroAchievements API is not enabled, skipping progress sync") + return None + + log.info("Scheduled RetroAchievements progress sync started...") + + users = db_user_handler.get_users(has_ra_username=True) + for user in users: + try: + user_progression = await meta_ra_handler.get_user_progression( + user.ra_username, + current_progression=cast( + RAUserProgression | None, user.ra_progression + ), + ) + db_user_handler.update_user( + user.id, + {"ra_progression": user_progression}, + ) + except Exception as e: + log.error( + f"Failed to update RetroAchievements progress for user: {user.username}, error: {e}" + ) + else: + log.debug( + f"Updated RetroAchievements progress for user: {user.username}" + ) + + log.info( + f"Scheduled RetroAchievements progress sync done. Updated users: {len(users)}" + ) + + +sync_retroachievements_progress_task = SyncRetroAchievementsProgressTask() diff --git a/backend/tests/tasks/test_sync_retroachievements_progress.py b/backend/tests/tasks/test_sync_retroachievements_progress.py new file mode 100644 index 000000000..c815e5b91 --- /dev/null +++ b/backend/tests/tasks/test_sync_retroachievements_progress.py @@ -0,0 +1,93 @@ +from unittest.mock import MagicMock, patch + +import pytest +from handler.database.users_handler import DBUsersHandler +from handler.metadata.ra_handler import RAHandler +from tasks.scheduled.sync_retroachievements_progress import ( + SyncRetroAchievementsProgressTask, +) + + +@pytest.fixture +def task() -> SyncRetroAchievementsProgressTask: + """Create a task instance for testing.""" + return SyncRetroAchievementsProgressTask() + + +class TestSyncRetroAchievementsProgressTask: + """Test suite for SyncRetroAchievementsProgressTask.""" + + def test_task_initialization(self, task): + """Test task initialization with correct parameters.""" + assert ( + task.func + == "tasks.scheduled.sync_retroachievements_progress.sync_retroachievements_progress_task.run" + ) + assert task.description == "Updates RetroAchievements progress for all users" + + @patch("tasks.scheduled.sync_retroachievements_progress.RA_API_ENABLED", False) + @patch("tasks.scheduled.sync_retroachievements_progress.log") + async def test_run_when_retroachievements_api_disabled(self, mock_log, task): + """Test run method when RetroAchievements API is disabled.""" + await task.run() + + mock_log.warning.assert_called_once_with( + "RetroAchievements API is not enabled, skipping progress sync" + ) + + async def test_run_when_no_users_set(self, task, mocker): + """Test run method when no users have RetroAchievements usernames set""" + mock_get_users = mocker.patch.object( + DBUsersHandler, "get_users", return_value=[] + ) + mock_get_user_progression = mocker.patch.object( + RAHandler, "get_user_progression" + ) + + await task.run() + + mock_get_users.assert_called_once_with(has_ra_username=True) + mock_get_user_progression.assert_not_called() + + async def test_run_saves_progress(self, task, viewer_user, mocker): + """Test run method saves retrieved progress.""" + mocker.patch.object(DBUsersHandler, "get_users", return_value=[viewer_user]) + mock_update_user = mocker.patch.object(DBUsersHandler, "update_user") + user_progression = MagicMock() + mocker.patch.object( + RAHandler, "get_user_progression", return_value=user_progression + ) + + await task.run() + + mock_update_user.assert_called_once_with( + viewer_user.id, + {"ra_progression": user_progression}, + ) + + async def test_run_is_resilient_to_errors( + self, task, viewer_user, editor_user, mocker + ): + """Test run method saves retrieved progress for a user even if another user fails.""" + mocker.patch.object( + DBUsersHandler, "get_users", return_value=[viewer_user, editor_user] + ) + user_progression = MagicMock() + mocker.patch.object( + RAHandler, + "get_user_progression", + side_effect=[ + # Call for first user raises an exception. + Exception("API error"), + # Call for second user returns valid progression. + user_progression, + ], + ) + mock_update_user = mocker.patch.object(DBUsersHandler, "update_user") + + await task.run() + + mock_update_user.assert_called_once_with( + editor_user.id, + {"ra_progression": user_progression}, + ) diff --git a/env.template b/env.template index 56b5b5546..05598941e 100644 --- a/env.template +++ b/env.template @@ -82,6 +82,8 @@ ENABLE_SCHEDULED_UPDATE_LAUNCHBOX_METADATA=true SCHEDULED_UPDATE_LAUNCHBOX_METADATA_CRON=0 4 * * * ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP=true SCHEDULED_CONVERT_IMAGES_TO_WEBP_CRON=0 4 * * * +ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC=true +SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON=0 4 * * * # In-browser emulation DISABLE_EMULATOR_JS=false