From 50199611ebb4d9c349d02808d36717bf0a46fc8d Mon Sep 17 00:00:00 2001
From: Georges-Antoine Assi
Date: Fri, 15 Sep 2023 13:11:41 -0400
Subject: [PATCH] Add mame xml and scheduled tasks
---
.gitignore | 1 +
backend/config/__init__.py | 8 ++++-
backend/endpoints/tests/test_heartbeat.py | 4 ++-
backend/handler/igdb_handler.py | 26 +++++++++++++++
backend/main.py | 4 +++
backend/scheduler.py | 2 ++
backend/tasks/update_mame_xml.py | 28 ++++++++++++++++
backend/tasks/update_switch_titledb.py | 31 +++--------------
backend/tasks/utils.py | 29 ++++++++++++++++
env.template | 2 ++
.../src/views/Settings/General/Settings.vue | 33 +++++++++++++++++++
poetry.lock | 17 +++++++++-
pyproject.toml | 1 +
pytest.ini | 2 ++
14 files changed, 159 insertions(+), 29 deletions(-)
create mode 100644 backend/tasks/update_mame_xml.py
diff --git a/.gitignore b/.gitignore
index 63e58fd16..0f06ff0e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -55,3 +55,4 @@ frontend/dev-dist
# outside data
switch_titledb.json
+mame.xml
diff --git a/backend/config/__init__.py b/backend/config/__init__.py
index 9a3f3b48b..bac5fc282 100644
--- a/backend/config/__init__.py
+++ b/backend/config/__init__.py
@@ -82,5 +82,11 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = (
os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true"
)
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
- "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 3 * * *" # At 3:00 AM every day
+ "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *" # At 4:00 AM every day
+)
+ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = (
+ os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true"
+)
+SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get(
+ "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *" # At 4:00 AM every day
)
diff --git a/backend/endpoints/tests/test_heartbeat.py b/backend/endpoints/tests/test_heartbeat.py
index 023a78d86..d095924a3 100644
--- a/backend/endpoints/tests/test_heartbeat.py
+++ b/backend/endpoints/tests/test_heartbeat.py
@@ -13,7 +13,9 @@ def test_heartbeat():
'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True,
'ENABLE_SCHEDULED_RESCAN': True,
'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True,
+ 'ENABLE_SCHEDULED_UPDATE_MAME_XML': True,
'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5,
'SCHEDULED_RESCAN_CRON': '0 3 * * *',
- 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 3 * * *',
+ 'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *',
+ 'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *',
}
diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py
index ceb3f84ca..048cd4359 100644
--- a/backend/handler/igdb_handler.py
+++ b/backend/handler/igdb_handler.py
@@ -6,6 +6,7 @@ import re
import time
import os
import json
+import xmltodict
from unidecode import unidecode as uc
from requests.exceptions import HTTPError, Timeout
from typing import Final
@@ -15,12 +16,14 @@ from utils import get_file_name_with_no_tags as get_search_term
from logger.logger import log
from utils.cache import cache
from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task
MAIN_GAME_CATEGORY: Final = 0
EXPANDED_GAME_CATEGORY: Final = 10
N_SCREENSHOTS: Final = 5
PS2_IGDB_ID: Final = 8
SWITCH_IGDB_ID: Final = 130
+ARCADE_IGDB_ID: Final = 52
PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
PS2_OPL_INDEX_FILE: Final = os.path.join(
@@ -32,6 +35,8 @@ SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_titledb.json"
)
+MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
+
class IGDBHandler:
def __init__(self) -> None:
@@ -183,6 +188,27 @@ class IGDBHandler:
if index_entry:
search_term = index_entry["name"] # type: ignore
+ if p_igdb_id == ARCADE_IGDB_ID:
+ mame_index = {}
+
+ try:
+ with open(MAME_XML_FILE, "r") as index_xml:
+ mame_index = xmltodict.parse(index_xml.read())
+ except FileNotFoundError:
+ log.warning("Fetching the MAME XML file from HyperspinFE...")
+ await update_mame_xml_task.run(force=True)
+
+ with open(MAME_XML_FILE, "r") as index_xml:
+ mame_index = xmltodict.parse(index_xml.read())
+ finally:
+ index_entry = [
+ game
+ for game in mame_index["menu"]["game"]
+ if game["@name"] == search_term
+ ]
+ if index_entry:
+ search_term = index_entry[0].get("description", search_term)
+
res = (
self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY)
or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY)
diff --git a/backend/main.py b/backend/main.py
index 666c8cb55..d8d6551a7 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -19,6 +19,8 @@ from config import (
SCHEDULED_RESCAN_CRON,
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+ ENABLE_SCHEDULED_UPDATE_MAME_XML,
+ SCHEDULED_UPDATE_MAME_XML_CRON,
)
from endpoints import search, platform, rom, identity, oauth, scan # noqa
from handler import dbh
@@ -82,6 +84,8 @@ def heartbeat():
"SCHEDULED_RESCAN_CRON": SCHEDULED_RESCAN_CRON,
"ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB": ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, # noqa
"SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON": SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+ "ENABLE_SCHEDULED_UPDATE_MAME_XML": ENABLE_SCHEDULED_UPDATE_MAME_XML,
+ "SCHEDULED_UPDATE_MAME_XML_CRON": SCHEDULED_UPDATE_MAME_XML_CRON,
}
diff --git a/backend/scheduler.py b/backend/scheduler.py
index 98b38625e..f36132a91 100644
--- a/backend/scheduler.py
+++ b/backend/scheduler.py
@@ -5,6 +5,7 @@ from tasks.utils import tasks_scheduler
from logger.logger import log
from tasks.scan_library import scan_library_task
from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task
if __name__ == "__main__":
if not ENABLE_EXPERIMENTAL_REDIS:
@@ -13,6 +14,7 @@ if __name__ == "__main__":
# Initialize the tasks
scan_library_task.init()
update_switch_titledb_task.init()
+ update_mame_xml_task.init()
log.info("Starting scheduler")
diff --git a/backend/tasks/update_mame_xml.py b/backend/tasks/update_mame_xml.py
new file mode 100644
index 000000000..23aa78910
--- /dev/null
+++ b/backend/tasks/update_mame_xml.py
@@ -0,0 +1,28 @@
+import os
+from pathlib import Path
+
+from typing import Final
+from config import (
+ ENABLE_SCHEDULED_UPDATE_MAME_XML,
+ SCHEDULED_UPDATE_MAME_XML_CRON,
+)
+from .utils import RemoteFilePullTask
+
+FIXTURE_FILE_PATH: Final = (
+ Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" / "mame.xml"
+)
+
+
+class UpdateMAMEXMLTask(RemoteFilePullTask):
+ def __init__(self):
+ super().__init__(
+ func="tasks.update_mame_xml.update_mame_xml_task.run",
+ description="mame xml update",
+ enabled=ENABLE_SCHEDULED_UPDATE_MAME_XML,
+ cron_string=SCHEDULED_UPDATE_MAME_XML_CRON,
+ url="https://hyperlist.hyperspin-fe.com/genall.php?system=6",
+ file_path=FIXTURE_FILE_PATH,
+ )
+
+
+update_mame_xml_task = UpdateMAMEXMLTask()
diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py
index 4f47cc6cd..eb48ba1b5 100644
--- a/backend/tasks/update_switch_titledb.py
+++ b/backend/tasks/update_switch_titledb.py
@@ -1,17 +1,14 @@
-import requests
import os
from pathlib import Path
-from logger.logger import log
from typing import Final
from config import (
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
)
-from .utils import PeriodicTask
+from .utils import RemoteFilePullTask
-RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json"
-FIXTURE_FILE_PATH = (
+FIXTURE_FILE_PATH: Final = (
Path(os.path.dirname(__file__)).parent
/ "handler"
/ "fixtures"
@@ -19,34 +16,16 @@ FIXTURE_FILE_PATH = (
)
-class UpdateSwitchTitleDBTask(PeriodicTask):
+class UpdateSwitchTitleDBTask(RemoteFilePullTask):
def __init__(self):
super().__init__(
func="tasks.update_switch_titledb.update_switch_titledb_task.run",
description="switch titledb update",
enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+ url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json",
+ file_path=FIXTURE_FILE_PATH,
)
- async def run(self, force: bool = False):
- if not ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB and not force:
- log.info("Scheduled switch titledb update not enabled, unscheduling...")
- self.unschedule()
- return
-
- log.info("Scheduled switch titledb update started...")
-
- try:
- response = requests.get(RAW_URL)
- response.raise_for_status()
-
- with open(FIXTURE_FILE_PATH, "wb") as fixture:
- fixture.write(response.content)
-
- log.info("Scheduled switch titledb update done")
- except requests.exceptions.RequestException as e:
- log.error("Scheduled switch titledb update failed", exc_info=True)
- log.error(e)
-
update_switch_titledb_task = UpdateSwitchTitleDBTask()
diff --git a/backend/tasks/utils.py b/backend/tasks/utils.py
index 96a3dcec8..140f18793 100644
--- a/backend/tasks/utils.py
+++ b/backend/tasks/utils.py
@@ -1,3 +1,4 @@
+import requests
from rq_scheduler import Scheduler
from abc import ABC, abstractmethod
@@ -73,3 +74,31 @@ class PeriodicTask(ABC):
tasks_scheduler.cancel(job)
log.info(f"{self.description.capitalize()} unscheduled.")
+
+
+class RemoteFilePullTask(PeriodicTask):
+ def __init__(self, *args, url: str, file_path: str, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.url = url
+ self.file_path = file_path
+
+ async def run(self, force: bool = False):
+ if not self.enabled and not force:
+ log.info(f"Scheduled {self.description} not enabled, unscheduling...")
+ self.unschedule()
+ return
+
+ log.info(f"Scheduled {self.description} started...")
+
+ try:
+ response = requests.get(self.url)
+ response.raise_for_status()
+
+ with open(self.file_path, "wb") as fixture:
+ fixture.write(response.content)
+
+ log.info(f"Scheduled {self.description} done")
+ except requests.exceptions.RequestException as e:
+ log.error(f"Scheduled {self.description} failed", exc_info=True)
+ log.error(e)
diff --git a/env.template b/env.template
index 374592386..1d3b7a1f0 100644
--- a/env.template
+++ b/env.template
@@ -39,3 +39,5 @@ ENABLE_SCHEDULED_RESCAN=true
SCHEDULED_RESCAN_CRON=0 3 * * *
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * *
+ENABLE_SCHEDULED_UPDATE_MAME_XML=true
+SCHEDULED_UPDATE_MAME_XML_CRON=0 4 * * *
diff --git a/frontend/src/views/Settings/General/Settings.vue b/frontend/src/views/Settings/General/Settings.vue
index c003db7cf..3777373e6 100644
--- a/frontend/src/views/Settings/General/Settings.vue
+++ b/frontend/src/views/Settings/General/Settings.vue
@@ -46,10 +46,17 @@ onBeforeMount(async () => {
switchUpdate =
switchUpdate.charAt(0).toLocaleLowerCase() + switchUpdate.substr(1);
+ let mameUpdate = cronstrue.toString(
+ data.SCHEDULED_UPDATE_MAME_XML_CRON,
+ { verbose: true }
+ );
+ mameUpdate = mameUpdate.charAt(0).toLocaleLowerCase() + mameUpdate.substr(1);
+
heartbeat.value = {
...data,
SCHEDULED_RESCAN_CRON: rescan,
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: switchUpdate,
+ SCHEDULED_UPDATE_MAME_XML_CRON: mameUpdate,
};
});
@@ -175,6 +182,32 @@ onBeforeMount(async () => {
+
+
+
+
+ Scheduled MAME XML update
+
+
+ Updates the Nintedo MAME XML file
+ {{ heartbeat.SCHEDULED_UPDATE_MAME_XML_CRON }}
+
+
+
diff --git a/poetry.lock b/poetry.lock
index 0aab17cab..900301abe 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -557,6 +557,7 @@ files = [
{file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
{file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
{file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
+ {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
{file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
{file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
{file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@@ -565,6 +566,7 @@ files = [
{file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
{file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
{file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
+ {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
{file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@@ -594,6 +596,7 @@ files = [
{file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
{file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
{file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
+ {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
{file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@@ -602,6 +605,7 @@ files = [
{file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
{file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
{file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
+ {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
{file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
{file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
{file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@@ -2180,6 +2184,17 @@ files = [
{file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"},
]
+[[package]]
+name = "xmltodict"
+version = "0.13.0"
+description = "Makes working with XML feel like you are working with JSON"
+optional = false
+python-versions = ">=3.4"
+files = [
+ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"},
+ {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
+]
+
[[package]]
name = "yarl"
version = "1.9.2"
@@ -2270,4 +2285,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
-content-hash = "29a5ef6641ea9b51f2b0fc389713b0316bd9feb8b3d89db341bc71449ccead61"
+content-hash = "2f2317b80e297bea8fde4735a5442778119a8022049a17edc853fceb79af2769"
diff --git a/pyproject.toml b/pyproject.toml
index d045a9579..79d694e59 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,6 +45,7 @@ python-multipart = "^0.0.6"
types-python-jose = "^3.3.4.8"
types-passlib = "^1.7.7.13"
watchdog = "^3.0.0"
+xmltodict = "^0.13.0"
[build-system]
requires = ["poetry-core"]
diff --git a/pytest.ini b/pytest.ini
index d39aa00d0..77f644468 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -13,6 +13,8 @@ env =
ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true
ENABLE_SCHEDULED_RESCAN=true
ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
+ ENABLE_SCHEDULED_UPDATE_MAME_XML=true
RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5
SCHEDULED_RESCAN_CRON=0 3 * * *
SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 3 * * *
+ SCHEDULED_UPDATE_MAME_XML_CRON=0 3 * * *