diff --git a/backend/adapters/services/rahasher.py b/backend/adapters/services/rahasher.py index f913a0458..6b8f20dc5 100644 --- a/backend/adapters/services/rahasher.py +++ b/backend/adapters/services/rahasher.py @@ -26,6 +26,8 @@ PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = { "colecovision": 44, "dreamcast": 40, "dc": 40, + "elektor": 75, + "fairchild-channel-f": 57, "gameboy": 4, "gb": 4, "gameboy-advance": 5, @@ -35,10 +37,11 @@ PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = { "game-gear": 15, "gamegear": 15, "gamecube": 16, - "ngc": 14, + "ngc": 16, "genesis": 1, - "genesis-slash-megadrive": 16, + "genesis-slash-megadrive": 1, "intellivision": 45, + "interton-vc-4000": 74, "jaguar": 17, "lynx": 13, "msx": 29, @@ -53,6 +56,7 @@ PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = { "nds": 18, "nintendo-dsi": 78, "odyssey-2": 23, + "win": 102, "pc-8000": 47, "pc-8800-series": 47, "pc-fx": 49, @@ -70,13 +74,16 @@ PLATFORM_SLUG_TO_RETROACHIEVEMENTS_ID: dict[str, int] = { "sms": 11, "sg-1000": 33, "snes": 3, + "sfam": 3, "turbografx-cd": 76, "turbografx-16-slash-pc-engine-cd": 76, "turbo-grafx": 8, "turbografx16--1": 8, - "vectrex": 26, + "uzebox": 80, + "vectrex": 46, "virtual-boy": 28, "virtualboy": 28, + "wasm-4": 72, "watara-slash-quickshot-supervision": 63, "wonderswan": 53, "wonderswan-color": 53, diff --git a/backend/adapters/services/retroachievements.py b/backend/adapters/services/retroachievements.py new file mode 100644 index 000000000..15fbbdda9 --- /dev/null +++ b/backend/adapters/services/retroachievements.py @@ -0,0 +1,185 @@ +import asyncio +import http +from typing import cast + +import aiohttp +import yarl +from adapters.services.retroachievements_types import ( + RAGameExtendedDetails, + RAGameInfoAndUserProgress, + RAGameListItem, + RAUserCompletionProgress, +) +from aiohttp.client import ClientTimeout +from config import RETROACHIEVEMENTS_API_KEY +from fastapi import HTTPException, status +from logger.logger import log +from utils.context import ctx_aiohttp_session + + +async def auth_middleware( + req: aiohttp.ClientRequest, handler: aiohttp.ClientHandlerType +) -> aiohttp.ClientResponse: + """RetroAchievements API authentication mechanism. + + Reference: https://api-docs.retroachievements.org/getting-started.html#quick-start-http-requests + """ + req.url = req.url.update_query({"y": RETROACHIEVEMENTS_API_KEY}) + return await handler(req) + + +class RetroAchievementsService: + """Service to interact with the RetroAchievements API. + + Reference: https://api-docs.retroachievements.org/ + """ + + def __init__( + self, + base_url: str | None = None, + ) -> None: + self.url = yarl.URL(base_url or "https://retroachievements.org/API") + + async def _request(self, url: str, request_timeout: int = 120) -> dict: + aiohttp_session = ctx_aiohttp_session.get() + log.debug( + "API request: URL=%s, Timeout=%s", + url, + request_timeout, + ) + try: + res = await aiohttp_session.get( + url, + middlewares=(auth_middleware,), + timeout=ClientTimeout(total=request_timeout), + ) + res.raise_for_status() + return await res.json() + except aiohttp.ServerTimeoutError: + # Retry the request once if it times out + pass + except aiohttp.ClientConnectionError as exc: + log.critical( + "Connection error: can't connect to RetroAchievements", exc_info=True + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Can't connect to RetroAchievements, check your internet connection", + ) from exc + except aiohttp.ClientResponseError as err: + if err.status == http.HTTPStatus.TOO_MANY_REQUESTS: + # Retry after 2 seconds if rate limit hit + await asyncio.sleep(2) + else: + # Log the error and return an empty dict if the request fails with a different code + log.error(err) + return {} + + try: + log.debug( + "API request: URL=%s, Timeout=%s", + url, + request_timeout, + ) + res = await aiohttp_session.get( + url, + middlewares=(auth_middleware,), + timeout=ClientTimeout(total=request_timeout), + ) + res.raise_for_status() + except (aiohttp.ClientResponseError, aiohttp.ServerTimeoutError) as err: + if ( + isinstance(err, aiohttp.ClientResponseError) + and err.status == http.HTTPStatus.UNAUTHORIZED + ): + return {} + + log.error(err) + return {} + + return await res.json() + + async def get_game_extended_details(self, game_id: int) -> RAGameExtendedDetails: + """Retrieve extended metadata about a game, targeted via its unique ID. + + Reference: https://api-docs.retroachievements.org/v1/get-game-extended.html + """ + url = self.url.joinpath("API_GetGameExtended.php").with_query( + i=[game_id], + ) + response = await self._request(str(url)) + return cast(RAGameExtendedDetails, response) + + async def get_game_list( + self, + system_id: int, + *, + only_games_with_achievements: bool = False, + include_hashes: bool = False, + limit: int | None = None, + offset: int | None = None, + ) -> list[RAGameListItem]: + """Retrieve the complete list of games for a specified console on the site, targeted by the console ID. + + Reference: https://api-docs.retroachievements.org/v1/get-game-list.html + """ + params: dict[str, list[str]] = {"i": [str(system_id)]} + if only_games_with_achievements: + params["f"] = ["1"] + if include_hashes: + params["h"] = ["1"] + if limit is not None: + params["c"] = [str(limit)] + if offset is not None: + params["o"] = [str(offset)] + + url = self.url.joinpath("API_GetGameList.php").with_query(**params) + response = await self._request(str(url)) + return cast(list[RAGameListItem], response) + + async def get_user_completion_progress( + self, + username: str, + *, + limit: int | None = None, + offset: int | None = None, + ) -> RAUserCompletionProgress: + """Retrieve a given user's completion progress, targeted by their username. + + Reference: https://api-docs.retroachievements.org/v1/get-user-completion-progress.html + """ + params: dict[str, list[str]] = {"u": [username]} + if limit is not None: + params["c"] = [str(limit)] + if offset is not None: + params["o"] = [str(offset)] + + url = self.url.joinpath("API_GetUserCompletionProgress.php").with_query( + **params + ) + response = await self._request(str(url)) + return cast(RAUserCompletionProgress, response) + + async def get_user_game_progress( + self, + username: str, + game_id: int, + *, + include_award_metadata: bool = False, + ) -> RAGameInfoAndUserProgress: + """Retrieve extended metadata about a game, in addition to a user's progress about that game. + + Reference: https://api-docs.retroachievements.org/v1/get-game-info-and-user-progress.html + """ + params: dict[str, list[str]] = { + "u": [username], + "g": [str(game_id)], + } + if include_award_metadata: + params["a"] = ["1"] + + url = self.url.joinpath("API_GetGameInfoAndUserProgress.php").with_query( + **params + ) + response = await self._request(str(url)) + return cast(RAGameInfoAndUserProgress, response) diff --git a/backend/adapters/services/retroachievements_types.py b/backend/adapters/services/retroachievements_types.py new file mode 100644 index 000000000..0f70d428c --- /dev/null +++ b/backend/adapters/services/retroachievements_types.py @@ -0,0 +1,164 @@ +import enum +from typing import NotRequired, TypedDict + + +# https://github.com/RetroAchievements/RAWeb/blob/master/app/Platform/Enums/AchievementType.php +class RAGameAchievementType(enum.StrEnum): + PROGRESSION = "progression" + WIN_CONDITION = "win_condition" + MISSABLE = "missable" + + +# https://github.com/RetroAchievements/RAWeb/blob/master/app/Platform/Enums/ReleasedAtGranularity.php +class RAGameReleasedAtGranularity(enum.StrEnum): + DAY = "day" + MONTH = "month" + YEAR = "year" + + +# https://github.com/RetroAchievements/RAWeb/blob/master/resources/js/common/components/AwardIndicator/AwardIndicator.tsx +class RAUserCompletionProgressKind(enum.StrEnum): + COMPLETED = "completed" + MASTERED = "mastered" + BEATEN_HARDCORE = "beaten-hardcore" + BEATEN_SOFTCORE = "beaten-softcore" + + +# https://api-docs.retroachievements.org/v1/get-game-extended.html#response +class RAGameExtendedDetailsAchievement(TypedDict): + ID: int + NumAwarded: int + NumAwardedHardcore: int + Title: str + Description: str + Points: int + TrueRatio: int + Author: str + AuthorULID: str + DateModified: str # ISO 8601 datetime format + DateCreated: str # ISO 8601 datetime format + BadgeName: str + DisplayOrder: int + MemAddr: str + type: RAGameAchievementType | None + + +# https://api-docs.retroachievements.org/v1/get-game-extended.html#response +class RAGameExtendedDetails(TypedDict): + ID: int + Title: str + ConsoleID: int + ForumTopicID: int | None + ImageIcon: str + ImageTitle: str + ImageIngame: str + ImageBoxArt: str + Publisher: str + Developer: str + Genre: str + Released: str # ISO 8601 date format + ReleasedAtGranularity: RAGameReleasedAtGranularity + RichPresencePatch: str + GuideURL: str | None + Updated: str # ISO 8601 datetime format + ConsoleName: str + ParentGameID: int | None + NumDistinctPlayers: int + NumAchievements: int + Achievements: dict[ + str, RAGameExtendedDetailsAchievement + ] # Key is achievement ID as string + NumDistinctPlayersCasual: int + NumDistinctPlayersHardcore: int + + +# https://api-docs.retroachievements.org/v1/get-user-completion-progress.html#response +class RAUserCompletionProgressResult(TypedDict): + GameID: int + Title: str + ImageIcon: str + ConsoleID: int + ConsoleName: str + MaxPossible: int + NumAwarded: int + NumAwardedHardcore: int + MostRecentAwardedDate: str # ISO 8601 datetime format + HighestAwardKind: RAUserCompletionProgressKind # e.g., "beaten-hardcore" + HighestAwardDate: str # ISO 8601 datetime format + + +# https://api-docs.retroachievements.org/v1/get-user-completion-progress.html#response +class RAUserCompletionProgress(TypedDict): + Count: int + Total: int + Results: list[RAUserCompletionProgressResult] + + +# https://api-docs.retroachievements.org/v1/get-game-info-and-user-progress.html#response +class RAGameInfoAndUserProgressAchievement(TypedDict): + ID: int + NumAwarded: int + NumAwardedHardcore: int + Title: str + Description: str + Points: int + TrueRatio: int + Author: str + AuthorULID: str + DateModified: str # ISO 8601 datetime format + DateCreated: str # ISO 8601 datetime format + BadgeName: str + DisplayOrder: int + MemAddr: str + type: RAGameAchievementType | None + DateEarnedHardcore: NotRequired[str] # ISO 8601 datetime format + DateEarned: NotRequired[str] # ISO 8601 datetime format + + +# https://api-docs.retroachievements.org/v1/get-game-info-and-user-progress.html#response +class RAGameInfoAndUserProgress(TypedDict): + ID: int + Title: str + ConsoleID: int + ForumTopicID: int | None + ImageIcon: str + ImageTitle: str + ImageIngame: str + ImageBoxArt: str + Publisher: str + Developer: str + Genre: str + Released: str # ISO 8601 date format + ReleasedAtGranularity: RAGameReleasedAtGranularity + RichPresencePatch: str + GuideURL: str | None + ConsoleName: str + ParentGameID: int | None + NumDistinctPlayers: int + NumAchievements: int + Achievements: dict[ + str, RAGameInfoAndUserProgressAchievement + ] # Key is achievement ID as string + NumAwardedToUser: int + NumAwardedToUserHardcore: int + NumDistinctPlayersCasual: int + NumDistinctPlayersHardcore: int + UserCompletion: str # e.g., "100.00%" + UserCompletionHardcore: str # e.g., "100.00%" + HighestAwardKind: NotRequired[RAUserCompletionProgressKind] + HighestAwardDate: NotRequired[str] # ISO 8601 datetime format + + +# https://api-docs.retroachievements.org/v1/get-game-list.html#response +class RAGameListItem(TypedDict): + Title: str + ID: int + ConsoleID: int + ConsoleName: str + ImageIcon: str + NumAchievements: int + NumLeaderboards: int + Points: int + DateModified: str # ISO 8601 datetime format + ForumTopicID: int | None + Hashes: NotRequired[list[str]] diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index f69b480e5..b289824a9 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -3,10 +3,13 @@ import http import json import os import time +from collections import defaultdict from typing import Final, NotRequired, TypedDict import httpx import yarl +from adapters.services.retroachievements import RetroAchievementsService +from adapters.services.retroachievements_types import RAGameListItem from anyio import open_file from config import ( REFRESH_RETROACHIEVEMENTS_CACHE_DAYS, @@ -76,15 +79,7 @@ class RAUserProgression(TypedDict): class RAHandler(MetadataHandler): def __init__(self) -> None: - self.BASE_URL = "https://retroachievements.org/API" - self.search_endpoint = f"{self.BASE_URL}/API_GetGameList.php" - self.game_details_endpoint = f"{self.BASE_URL}/API_GetGameExtended.php" - self.user_complete_progression_endpoint = ( - f"{self.BASE_URL}/API_GetUserCompletionProgress.php" - ) - self.user_game_progression_endpoint = ( - f"{self.BASE_URL}/API_GetGameInfoAndUserProgress.php" - ) + self.ra_service = RetroAchievementsService() self.HASHES_FILE_NAME = "ra_hashes.json" def _get_rom_base_path(self, platform_id: int, rom_id: int) -> str: @@ -185,28 +180,26 @@ class RAHandler(MetadataHandler): async def _search_rom( self, platform: Platform, rom_id: int, hash: str - ) -> dict | None: + ) -> RAGameListItem | None: if not platform.ra_id: return None self._create_resources_path(platform.id, rom_id) - url = yarl.URL(self.search_endpoint).with_query( - i=[platform.ra_id], - f=["1"], # If 1, only return games that have achievements. Defaults to 0. - h=["1"], # If 1, also return supported hashes for games. Defaults to 0. - y=[RETROACHIEVEMENTS_API_KEY], - ) - # Fetch all hashes for specific platform + roms: list[RAGameListItem] if ( REFRESH_RETROACHIEVEMENTS_CACHE_DAYS <= self._days_since_last_cache_file_update(platform.id) or not self._exists_cache_file(platform.id) ): # Write the roms result to a JSON file if older than REFRESH_RETROACHIEVEMENTS_CACHE_DAYS days - roms = await self._request(str(url)) + roms = await self.ra_service.get_game_list( + system_id=platform.ra_id, + only_games_with_achievements=True, + include_hashes=True, + ) async with await open_file( self._get_hashes_file_path(platform.id), "w", @@ -223,19 +216,11 @@ class RAHandler(MetadataHandler): roms = json.loads(await json_file.read()) for rom in roms: - if hash in rom["Hashes"]: + if hash in rom.get("Hashes", ()): return rom return None - async def _get_rom_details(self, ra_id: int) -> dict: - url = yarl.URL(self.game_details_endpoint).with_query( - i=[ra_id], - y=[RETROACHIEVEMENTS_API_KEY], - ) - details = await self._request(str(url)) - return details - def get_platform(self, slug: str) -> RAGamesPlatform: platform = SLUG_TO_RA_ID.get(slug.lower(), None) @@ -258,7 +243,7 @@ class RAHandler(MetadataHandler): return RAGameRom(ra_id=None) try: - rom_details = await self._get_rom_details(rom["ID"]) + rom_details = await self.ra_service.get_game_extended_details(rom["ID"]) return RAGameRom( ra_id=rom["ID"], ra_metadata=RAMetadata( @@ -288,25 +273,24 @@ class RAHandler(MetadataHandler): return RAGameRom(ra_id=None) async def get_user_progression(self, username: str) -> RAUserProgression: - url = yarl.URL(self.user_complete_progression_endpoint).with_query( - u=[username], y=[RETROACHIEVEMENTS_API_KEY], c=[500] + user_complete_progression = await self.ra_service.get_user_completion_progress( + username=username, + limit=500, ) - user_complete_progression = await self._request(str(url)) roms_with_progression = user_complete_progression.get("Results", []) + rom_earned_achievements: dict[int, list[EarnedAchievement]] = defaultdict(list) for rom in roms_with_progression: - rom["EarnedAchievements"] = [] - if rom.get("GameID", None): - url = yarl.URL(self.user_game_progression_endpoint).with_query( - g=[rom["GameID"]], - u=[username], - y=[RETROACHIEVEMENTS_API_KEY], + rom_game_id = rom.get("GameID") + if rom_game_id: + result = await self.ra_service.get_user_game_progress( + username=username, + game_id=rom_game_id, ) - result = await self._request(str(url)) for achievement in result.get("Achievements", {}).values(): - if "DateEarned" in achievement.keys(): - rom["EarnedAchievements"].append( + if achievement.get("DateEarned") and achievement.get("BadgeName"): + rom_earned_achievements[rom_game_id].append( { - "id": achievement.get("BadgeName"), + "id": achievement["BadgeName"], "date": achievement["DateEarned"], } ) @@ -319,7 +303,11 @@ class RAHandler(MetadataHandler): max_possible=rom.get("MaxPossible", None), num_awarded=rom.get("NumAwarded", None), num_awarded_hardcore=rom.get("NumAwardedHardcore", None), - earned_achievements=rom.get("EarnedAchievements", []), + earned_achievements=( + rom_earned_achievements.get(rom["GameID"], []) + if rom.get("GameID") + else [] + ), ) for rom in roms_with_progression ], diff --git a/backend/main.py b/backend/main.py index 7636bc181..e55ba5d07 100644 --- a/backend/main.py +++ b/backend/main.py @@ -43,7 +43,12 @@ from handler.socket_handler import socket_handler from logger.log_middleware import LOGGING_CONFIG, CustomLoggingMiddleware from starlette.middleware.authentication import AuthenticationMiddleware from utils import get_version -from utils.context import ctx_httpx_client, initialize_context, set_context_middleware +from utils.context import ( + ctx_aiohttp_session, + ctx_httpx_client, + initialize_context, + set_context_middleware, +) logging.config.dictConfig(LOGGING_CONFIG) @@ -51,6 +56,7 @@ logging.config.dictConfig(LOGGING_CONFIG) @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: async with initialize_context(): + app.state.aiohttp_session = ctx_aiohttp_session.get() app.state.httpx_client = ctx_httpx_client.get() yield diff --git a/backend/utils/context.py b/backend/utils/context.py index 817de73ee..bc8835174 100644 --- a/backend/utils/context.py +++ b/backend/utils/context.py @@ -3,11 +3,13 @@ from contextlib import asynccontextmanager from contextvars import ContextVar, Token from typing import TypeVar +import aiohttp import httpx from fastapi import Request, Response _T = TypeVar("_T") +ctx_aiohttp_session: ContextVar[aiohttp.ClientSession] = ContextVar("aiohttp_session") ctx_httpx_client: ContextVar[httpx.AsyncClient] = ContextVar("httpx_client") @@ -24,9 +26,13 @@ async def set_context_var( @asynccontextmanager async def initialize_context() -> AsyncGenerator[None, None]: """Initialize context variables.""" - async with httpx.AsyncClient() as httpx_client: - async with set_context_var(ctx_httpx_client, httpx_client): - yield + async with ( + aiohttp.ClientSession() as aiohttp_session, + httpx.AsyncClient() as httpx_client, + set_context_var(ctx_aiohttp_session, aiohttp_session), + set_context_var(ctx_httpx_client, httpx_client), + ): + yield async def set_context_middleware( @@ -37,5 +43,8 @@ async def set_context_middleware( This middleware is needed because the context initialized during the lifespan process is not available in the request-response cycle. """ - async with set_context_var(ctx_httpx_client, request.app.state.httpx_client): + async with ( + set_context_var(ctx_aiohttp_session, request.app.state.aiohttp_session), + set_context_var(ctx_httpx_client, request.app.state.httpx_client), + ): return await call_next(request) diff --git a/frontend/src/components/Gallery/AppBar/Search/SearchTextField.vue b/frontend/src/components/Gallery/AppBar/Search/SearchTextField.vue index 9f75ca0a0..a4394b5ef 100644 --- a/frontend/src/components/Gallery/AppBar/Search/SearchTextField.vue +++ b/frontend/src/components/Gallery/AppBar/Search/SearchTextField.vue @@ -5,17 +5,15 @@ import type { Events } from "@/types/emitter"; import type { Emitter } from "mitt"; import { inject, nextTick, onMounted, watch } from "vue"; import { storeToRefs } from "pinia"; -import { useDisplay } from "vuetify"; import { useRouter } from "vue-router"; import { useI18n } from "vue-i18n"; import { debounce } from "lodash"; // Props -const { xs } = useDisplay(); const { t } = useI18n(); const router = useRouter(); const romsStore = storeRoms(); -const { fetchingRoms, initialSearch } = storeToRefs(romsStore); +const { initialSearch } = storeToRefs(romsStore); const emitter = inject>("emitter"); const galleryFilterStore = storeGalleryFilter(); const { searchTerm } = storeToRefs(galleryFilterStore); @@ -98,6 +96,7 @@ onMounted(() => { // Check if search term is set in the URL (empty string is ok) if (searchTerm.value !== null) { + romsStore.resetPagination(); fetchRoms(); } }); @@ -107,6 +106,7 @@ watch( (query) => { if (query.search !== undefined && query.search !== searchTerm.value) { searchTerm.value = query.search as string; + romsStore.resetPagination(); fetchRoms(); } }, diff --git a/frontend/src/components/common/Game/Card/ActionBar.vue b/frontend/src/components/common/Game/Card/ActionBar.vue index b5e3b88ae..f7fff5f9d 100644 --- a/frontend/src/components/common/Game/Card/ActionBar.vue +++ b/frontend/src/components/common/Game/Card/ActionBar.vue @@ -57,7 +57,7 @@ watch(menuOpen, (val) => {