From 50199611ebb4d9c349d02808d36717bf0a46fc8d Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Fri, 15 Sep 2023 13:11:41 -0400 Subject: [PATCH] Add mame xml and scheduled tasks --- .gitignore | 1 + backend/config/__init__.py | 8 ++++- backend/endpoints/tests/test_heartbeat.py | 4 ++- backend/handler/igdb_handler.py | 26 +++++++++++++++ backend/main.py | 4 +++ backend/scheduler.py | 2 ++ backend/tasks/update_mame_xml.py | 28 ++++++++++++++++ backend/tasks/update_switch_titledb.py | 31 +++-------------- backend/tasks/utils.py | 29 ++++++++++++++++ env.template | 2 ++ .../src/views/Settings/General/Settings.vue | 33 +++++++++++++++++++ poetry.lock | 17 +++++++++- pyproject.toml | 1 + pytest.ini | 2 ++ 14 files changed, 159 insertions(+), 29 deletions(-) create mode 100644 backend/tasks/update_mame_xml.py diff --git a/.gitignore b/.gitignore index 63e58fd16..0f06ff0e0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ frontend/dev-dist # outside data switch_titledb.json +mame.xml diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 9a3f3b48b..bac5fc282 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -82,5 +82,11 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = ( os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true" ) SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get( - "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 3 * * *" # At 3:00 AM every day + "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *" # At 4:00 AM every day +) +ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = ( + os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true" +) +SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get( + "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *" # At 4:00 AM every day ) diff --git a/backend/endpoints/tests/test_heartbeat.py b/backend/endpoints/tests/test_heartbeat.py index 023a78d86..d095924a3 100644 --- a/backend/endpoints/tests/test_heartbeat.py +++ b/backend/endpoints/tests/test_heartbeat.py @@ -13,7 +13,9 @@ def test_heartbeat(): 'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True, 'ENABLE_SCHEDULED_RESCAN': True, 'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True, + 'ENABLE_SCHEDULED_UPDATE_MAME_XML': True, 'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5, 'SCHEDULED_RESCAN_CRON': '0 3 * * *', - 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 3 * * *', + 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *', + 'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *', } diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index ceb3f84ca..048cd4359 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -6,6 +6,7 @@ import re import time import os import json +import xmltodict from unidecode import unidecode as uc from requests.exceptions import HTTPError, Timeout from typing import Final @@ -15,12 +16,14 @@ from utils import get_file_name_with_no_tags as get_search_term from logger.logger import log from utils.cache import cache from tasks.update_switch_titledb import update_switch_titledb_task +from tasks.update_mame_xml import update_mame_xml_task MAIN_GAME_CATEGORY: Final = 0 EXPANDED_GAME_CATEGORY: Final = 10 N_SCREENSHOTS: Final = 5 PS2_IGDB_ID: Final = 8 SWITCH_IGDB_ID: Final = 130 +ARCADE_IGDB_ID: Final = 52 PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$" PS2_OPL_INDEX_FILE: Final = os.path.join( @@ -32,6 +35,8 @@ SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join( os.path.dirname(__file__), "fixtures", "switch_titledb.json" ) +MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml") + class IGDBHandler: def __init__(self) -> None: @@ -183,6 +188,27 @@ class IGDBHandler: if index_entry: search_term = index_entry["name"] # type: ignore + if p_igdb_id == ARCADE_IGDB_ID: + mame_index = {} + + try: + with open(MAME_XML_FILE, "r") as index_xml: + mame_index = xmltodict.parse(index_xml.read()) + except FileNotFoundError: + log.warning("Fetching the MAME XML file from HyperspinFE...") + await update_mame_xml_task.run(force=True) + + with open(MAME_XML_FILE, "r") as index_xml: + mame_index = xmltodict.parse(index_xml.read()) + finally: + index_entry = [ + game + for game in mame_index["menu"]["game"] + if game["@name"] == search_term + ] + if index_entry: + search_term = index_entry[0].get("description", search_term) + res = ( self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY) or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY) diff --git a/backend/main.py b/backend/main.py index 666c8cb55..d8d6551a7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -19,6 +19,8 @@ from config import ( SCHEDULED_RESCAN_CRON, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + ENABLE_SCHEDULED_UPDATE_MAME_XML, + SCHEDULED_UPDATE_MAME_XML_CRON, ) from endpoints import search, platform, rom, identity, oauth, scan # noqa from handler import dbh @@ -82,6 +84,8 @@ def heartbeat(): "SCHEDULED_RESCAN_CRON": SCHEDULED_RESCAN_CRON, "ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB": ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, # noqa "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON": SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + "ENABLE_SCHEDULED_UPDATE_MAME_XML": ENABLE_SCHEDULED_UPDATE_MAME_XML, + "SCHEDULED_UPDATE_MAME_XML_CRON": SCHEDULED_UPDATE_MAME_XML_CRON, } diff --git a/backend/scheduler.py b/backend/scheduler.py index 98b38625e..f36132a91 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -5,6 +5,7 @@ from tasks.utils import tasks_scheduler from logger.logger import log from tasks.scan_library import scan_library_task from tasks.update_switch_titledb import update_switch_titledb_task +from tasks.update_mame_xml import update_mame_xml_task if __name__ == "__main__": if not ENABLE_EXPERIMENTAL_REDIS: @@ -13,6 +14,7 @@ if __name__ == "__main__": # Initialize the tasks scan_library_task.init() update_switch_titledb_task.init() + update_mame_xml_task.init() log.info("Starting scheduler") diff --git a/backend/tasks/update_mame_xml.py b/backend/tasks/update_mame_xml.py new file mode 100644 index 000000000..23aa78910 --- /dev/null +++ b/backend/tasks/update_mame_xml.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path + +from typing import Final +from config import ( + ENABLE_SCHEDULED_UPDATE_MAME_XML, + SCHEDULED_UPDATE_MAME_XML_CRON, +) +from .utils import RemoteFilePullTask + +FIXTURE_FILE_PATH: Final = ( + Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" / "mame.xml" +) + + +class UpdateMAMEXMLTask(RemoteFilePullTask): + def __init__(self): + super().__init__( + func="tasks.update_mame_xml.update_mame_xml_task.run", + description="mame xml update", + enabled=ENABLE_SCHEDULED_UPDATE_MAME_XML, + cron_string=SCHEDULED_UPDATE_MAME_XML_CRON, + url="https://hyperlist.hyperspin-fe.com/genall.php?system=6", + file_path=FIXTURE_FILE_PATH, + ) + + +update_mame_xml_task = UpdateMAMEXMLTask() diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py index 4f47cc6cd..eb48ba1b5 100644 --- a/backend/tasks/update_switch_titledb.py +++ b/backend/tasks/update_switch_titledb.py @@ -1,17 +1,14 @@ -import requests import os from pathlib import Path -from logger.logger import log from typing import Final from config import ( ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, ) -from .utils import PeriodicTask +from .utils import RemoteFilePullTask -RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json" -FIXTURE_FILE_PATH = ( +FIXTURE_FILE_PATH: Final = ( Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" @@ -19,34 +16,16 @@ FIXTURE_FILE_PATH = ( ) -class UpdateSwitchTitleDBTask(PeriodicTask): +class UpdateSwitchTitleDBTask(RemoteFilePullTask): def __init__(self): super().__init__( func="tasks.update_switch_titledb.update_switch_titledb_task.run", description="switch titledb update", enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON, + url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json", + file_path=FIXTURE_FILE_PATH, ) - async def run(self, force: bool = False): - if not ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB and not force: - log.info("Scheduled switch titledb update not enabled, unscheduling...") - self.unschedule() - return - - log.info("Scheduled switch titledb update started...") - - try: - response = requests.get(RAW_URL) - response.raise_for_status() - - with open(FIXTURE_FILE_PATH, "wb") as fixture: - fixture.write(response.content) - - log.info("Scheduled switch titledb update done") - except requests.exceptions.RequestException as e: - log.error("Scheduled switch titledb update failed", exc_info=True) - log.error(e) - update_switch_titledb_task = UpdateSwitchTitleDBTask() diff --git a/backend/tasks/utils.py b/backend/tasks/utils.py index 96a3dcec8..140f18793 100644 --- a/backend/tasks/utils.py +++ b/backend/tasks/utils.py @@ -1,3 +1,4 @@ +import requests from rq_scheduler import Scheduler from abc import ABC, abstractmethod @@ -73,3 +74,31 @@ class PeriodicTask(ABC): tasks_scheduler.cancel(job) log.info(f"{self.description.capitalize()} unscheduled.") + + +class RemoteFilePullTask(PeriodicTask): + def __init__(self, *args, url: str, file_path: str, **kwargs): + super().__init__(*args, **kwargs) + + self.url = url + self.file_path = file_path + + async def run(self, force: bool = False): + if not self.enabled and not force: + log.info(f"Scheduled {self.description} not enabled, unscheduling...") + self.unschedule() + return + + log.info(f"Scheduled {self.description} started...") + + try: + response = requests.get(self.url) + response.raise_for_status() + + with open(self.file_path, "wb") as fixture: + fixture.write(response.content) + + log.info(f"Scheduled {self.description} done") + except requests.exceptions.RequestException as e: + log.error(f"Scheduled {self.description} failed", exc_info=True) + log.error(e) diff --git a/env.template b/env.template index 374592386..1d3b7a1f0 100644 --- a/env.template +++ b/env.template @@ -39,3 +39,5 @@ ENABLE_SCHEDULED_RESCAN=true SCHEDULED_RESCAN_CRON=0 3 * * * ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * * +ENABLE_SCHEDULED_UPDATE_MAME_XML=true +SCHEDULED_UPDATE_MAME_XML_CRON=0 4 * * * diff --git a/frontend/src/views/Settings/General/Settings.vue b/frontend/src/views/Settings/General/Settings.vue index c003db7cf..3777373e6 100644 --- a/frontend/src/views/Settings/General/Settings.vue +++ b/frontend/src/views/Settings/General/Settings.vue @@ -46,10 +46,17 @@ onBeforeMount(async () => { switchUpdate = switchUpdate.charAt(0).toLocaleLowerCase() + switchUpdate.substr(1); + let mameUpdate = cronstrue.toString( + data.SCHEDULED_UPDATE_MAME_XML_CRON, + { verbose: true } + ); + mameUpdate = mameUpdate.charAt(0).toLocaleLowerCase() + mameUpdate.substr(1); + heartbeat.value = { ...data, SCHEDULED_RESCAN_CRON: rescan, SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: switchUpdate, + SCHEDULED_UPDATE_MAME_XML_CRON: mameUpdate, }; }); @@ -175,6 +182,32 @@ onBeforeMount(async () => {

+ + +
+ + Scheduled MAME XML update + +

+ Updates the Nintedo MAME XML file + {{ heartbeat.SCHEDULED_UPDATE_MAME_XML_CRON }} +

+
+
diff --git a/poetry.lock b/poetry.lock index 0aab17cab..900301abe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -557,6 +557,7 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -565,6 +566,7 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -594,6 +596,7 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -602,6 +605,7 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -2180,6 +2184,17 @@ files = [ {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + [[package]] name = "yarl" version = "1.9.2" @@ -2270,4 +2285,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "29a5ef6641ea9b51f2b0fc389713b0316bd9feb8b3d89db341bc71449ccead61" +content-hash = "2f2317b80e297bea8fde4735a5442778119a8022049a17edc853fceb79af2769" diff --git a/pyproject.toml b/pyproject.toml index d045a9579..79d694e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ python-multipart = "^0.0.6" types-python-jose = "^3.3.4.8" types-passlib = "^1.7.7.13" watchdog = "^3.0.0" +xmltodict = "^0.13.0" [build-system] requires = ["poetry-core"] diff --git a/pytest.ini b/pytest.ini index d39aa00d0..77f644468 100644 --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,8 @@ env = ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true ENABLE_SCHEDULED_RESCAN=true ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true + ENABLE_SCHEDULED_UPDATE_MAME_XML=true RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5 SCHEDULED_RESCAN_CRON=0 3 * * * SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 3 * * * + SCHEDULED_UPDATE_MAME_XML_CRON=0 3 * * *