diff --git a/.gitignore b/.gitignore index 63e58fd16..0f06ff0e0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ frontend/dev-dist # outside data switch_titledb.json +mame.xml diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 9a3f3b48b..bac5fc282 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -82,5 +82,11 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = ( os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true" ) SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get( - "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 3 * * *" # At 3:00 AM every day + "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *" # At 4:00 AM every day +) +ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = ( + os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true" +) +SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get( + "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *" # At 4:00 AM every day ) diff --git a/backend/endpoints/tests/test_heartbeat.py b/backend/endpoints/tests/test_heartbeat.py index 023a78d86..d095924a3 100644 --- a/backend/endpoints/tests/test_heartbeat.py +++ b/backend/endpoints/tests/test_heartbeat.py @@ -13,7 +13,9 @@ def test_heartbeat(): 'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True, 'ENABLE_SCHEDULED_RESCAN': True, 'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True, + 'ENABLE_SCHEDULED_UPDATE_MAME_XML': True, 'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5, 'SCHEDULED_RESCAN_CRON': '0 3 * * *', - 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 3 * * *', + 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *', + 'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *', } diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index ceb3f84ca..048cd4359 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -6,6 +6,7 @@ import re import time import os import json +import xmltodict from unidecode import unidecode as uc from requests.exceptions import HTTPError, Timeout from typing import Final @@ -15,12 +16,14 @@ from utils import get_file_name_with_no_tags as get_search_term from logger.logger import log from utils.cache import cache from tasks.update_switch_titledb import update_switch_titledb_task +from tasks.update_mame_xml import update_mame_xml_task MAIN_GAME_CATEGORY: Final = 0 EXPANDED_GAME_CATEGORY: Final = 10 N_SCREENSHOTS: Final = 5 PS2_IGDB_ID: Final = 8 SWITCH_IGDB_ID: Final = 130 +ARCADE_IGDB_ID: Final = 52 PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$" PS2_OPL_INDEX_FILE: Final = os.path.join( @@ -32,6 +35,8 @@ SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join( os.path.dirname(__file__), "fixtures", "switch_titledb.json" ) +MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml") + class IGDBHandler: def __init__(self) -> None: @@ -183,6 +188,27 @@ class IGDBHandler: if index_entry: search_term = index_entry["name"] # type: ignore + if p_igdb_id == ARCADE_IGDB_ID: + mame_index = {} + + try: + with open(MAME_XML_FILE, "r") as index_xml: + mame_index = xmltodict.parse(index_xml.read()) + except FileNotFoundError: + log.warning("Fetching the MAME XML file from HyperspinFE...") + await update_mame_xml_task.run(force=True) + + with open(MAME_XML_FILE, "r") as index_xml: + mame_index = xmltodict.parse(index_xml.read()) + finally: + index_entry = [ + game + for game in mame_index["menu"]["game"] + if game["@name"] == search_term + ] + if index_entry: + search_term = index_entry[0].get("description", search_term) + res = ( self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY) or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY) diff --git a/backend/main.py b/backend/main.py index 666c8cb55..d8d6551a7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -19,6 +19,8 @@ from config import ( SCHEDULED_RESCAN_CRON, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + ENABLE_SCHEDULED_UPDATE_MAME_XML, + SCHEDULED_UPDATE_MAME_XML_CRON, ) from endpoints import search, platform, rom, identity, oauth, scan # noqa from handler import dbh @@ -82,6 +84,8 @@ def heartbeat(): "SCHEDULED_RESCAN_CRON": SCHEDULED_RESCAN_CRON, "ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB": ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, # noqa "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON": SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + "ENABLE_SCHEDULED_UPDATE_MAME_XML": ENABLE_SCHEDULED_UPDATE_MAME_XML, + "SCHEDULED_UPDATE_MAME_XML_CRON": SCHEDULED_UPDATE_MAME_XML_CRON, } diff --git a/backend/scheduler.py b/backend/scheduler.py index 98b38625e..f36132a91 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -5,6 +5,7 @@ from tasks.utils import tasks_scheduler from logger.logger import log from tasks.scan_library import scan_library_task from tasks.update_switch_titledb import update_switch_titledb_task +from tasks.update_mame_xml import update_mame_xml_task if __name__ == "__main__": if not ENABLE_EXPERIMENTAL_REDIS: @@ -13,6 +14,7 @@ if __name__ == "__main__": # Initialize the tasks scan_library_task.init() update_switch_titledb_task.init() + update_mame_xml_task.init() log.info("Starting scheduler") diff --git a/backend/tasks/update_mame_xml.py b/backend/tasks/update_mame_xml.py new file mode 100644 index 000000000..23aa78910 --- /dev/null +++ b/backend/tasks/update_mame_xml.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path + +from typing import Final +from config import ( + ENABLE_SCHEDULED_UPDATE_MAME_XML, + SCHEDULED_UPDATE_MAME_XML_CRON, +) +from .utils import RemoteFilePullTask + +FIXTURE_FILE_PATH: Final = ( + Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" / "mame.xml" +) + + +class UpdateMAMEXMLTask(RemoteFilePullTask): + def __init__(self): + super().__init__( + func="tasks.update_mame_xml.update_mame_xml_task.run", + description="mame xml update", + enabled=ENABLE_SCHEDULED_UPDATE_MAME_XML, + cron_string=SCHEDULED_UPDATE_MAME_XML_CRON, + url="https://hyperlist.hyperspin-fe.com/genall.php?system=6", + file_path=FIXTURE_FILE_PATH, + ) + + +update_mame_xml_task = UpdateMAMEXMLTask() diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py index 4f47cc6cd..eb48ba1b5 100644 --- a/backend/tasks/update_switch_titledb.py +++ b/backend/tasks/update_switch_titledb.py @@ -1,17 +1,14 @@ -import requests import os from pathlib import Path -from logger.logger import log from typing import Final from config import ( ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, ) -from .utils import PeriodicTask +from .utils import RemoteFilePullTask -RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json" -FIXTURE_FILE_PATH = ( +FIXTURE_FILE_PATH: Final = ( Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" @@ -19,34 +16,16 @@ FIXTURE_FILE_PATH = ( ) -class UpdateSwitchTitleDBTask(PeriodicTask): +class UpdateSwitchTitleDBTask(RemoteFilePullTask): def __init__(self): super().__init__( func="tasks.update_switch_titledb.update_switch_titledb_task.run", description="switch titledb update", enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json", + file_path=FIXTURE_FILE_PATH, ) - async def run(self, force: bool = False): - if not ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB and not force: - log.info("Scheduled switch titledb update not enabled, unscheduling...") - self.unschedule() - return - - log.info("Scheduled switch titledb update started...") - - try: - response = requests.get(RAW_URL) - response.raise_for_status() - - with open(FIXTURE_FILE_PATH, "wb") as fixture: - fixture.write(response.content) - - log.info("Scheduled switch titledb update done") - except requests.exceptions.RequestException as e: - log.error("Scheduled switch titledb update failed", exc_info=True) - log.error(e) - update_switch_titledb_task = UpdateSwitchTitleDBTask() diff --git a/backend/tasks/utils.py b/backend/tasks/utils.py index 96a3dcec8..140f18793 100644 --- a/backend/tasks/utils.py +++ b/backend/tasks/utils.py @@ -1,3 +1,4 @@ +import requests from rq_scheduler import Scheduler from abc import ABC, abstractmethod @@ -73,3 +74,31 @@ class PeriodicTask(ABC): tasks_scheduler.cancel(job) log.info(f"{self.description.capitalize()} unscheduled.") + + +class RemoteFilePullTask(PeriodicTask): + def __init__(self, *args, url: str, file_path: str, **kwargs): + super().__init__(*args, **kwargs) + + self.url = url + self.file_path = file_path + + async def run(self, force: bool = False): + if not self.enabled and not force: + log.info(f"Scheduled {self.description} not enabled, unscheduling...") + self.unschedule() + return + + log.info(f"Scheduled {self.description} started...") + + try: + response = requests.get(self.url) + response.raise_for_status() + + with open(self.file_path, "wb") as fixture: + fixture.write(response.content) + + log.info(f"Scheduled {self.description} done") + except requests.exceptions.RequestException as e: + log.error(f"Scheduled {self.description} failed", exc_info=True) + log.error(e) diff --git a/env.template b/env.template index 374592386..1d3b7a1f0 100644 --- a/env.template +++ b/env.template @@ -39,3 +39,5 @@ ENABLE_SCHEDULED_RESCAN=true SCHEDULED_RESCAN_CRON=0 3 * * * ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * * +ENABLE_SCHEDULED_UPDATE_MAME_XML=true +SCHEDULED_UPDATE_MAME_XML_CRON=0 4 * * * diff --git a/frontend/src/views/Settings/General/Settings.vue b/frontend/src/views/Settings/General/Settings.vue index c003db7cf..3777373e6 100644 --- a/frontend/src/views/Settings/General/Settings.vue +++ b/frontend/src/views/Settings/General/Settings.vue @@ -46,10 +46,17 @@ onBeforeMount(async () => { switchUpdate = switchUpdate.charAt(0).toLocaleLowerCase() + switchUpdate.substr(1); + let mameUpdate = cronstrue.toString( + data.SCHEDULED_UPDATE_MAME_XML_CRON, + { verbose: true } + ); + mameUpdate = mameUpdate.charAt(0).toLocaleLowerCase() + mameUpdate.substr(1); + heartbeat.value = { ...data, SCHEDULED_RESCAN_CRON: rescan, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: switchUpdate, + SCHEDULED_UPDATE_MAME_XML_CRON: mameUpdate, }; }); @@ -175,6 +182,32 @@ onBeforeMount(async () => {
++ Updates the Nintedo MAME XML file + {{ heartbeat.SCHEDULED_UPDATE_MAME_XML_CRON }} +
+