Add mame xml and scheduled tasks

2026-02-18 23:42:07 +01:00 · 2023-09-15 13:11:41 -04:00
parent 0dd2511039
commit 50199611eb
14 changed files with 159 additions and 29 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -55,3 +55,4 @@ frontend/dev-dist

 # outside data
 switch_titledb.json
+mame.xml
--- a/backend/config/init.py
+++ b/backend/config/init.py
@@ -82,5 +82,11 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: Final = (
    os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true"
 )
 SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
-    "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 3 * * *"  # At 3:00 AM every day
+    "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *"  # At 4:00 AM every day
+)
+ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = (
+    os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true"
+)
+SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get(
+    "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *"  # At 4:00 AM every day
 )
--- a/backend/endpoints/tests/test_heartbeat.py
+++ b/backend/endpoints/tests/test_heartbeat.py
@@ -13,7 +13,9 @@ def test_heartbeat():
        'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True,
        'ENABLE_SCHEDULED_RESCAN': True,
        'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True,
+        'ENABLE_SCHEDULED_UPDATE_MAME_XML': True,
        'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5,
        'SCHEDULED_RESCAN_CRON': '0 3 * * *',
-        'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 3 * * *',
+        'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *',
+        'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *',
    }
--- a/backend/handler/igdb_handler.py
+++ b/backend/handler/igdb_handler.py
@@ -6,6 +6,7 @@ import re
 import time
 import os
 import json
+import xmltodict
 from unidecode import unidecode as uc
 from requests.exceptions import HTTPError, Timeout
 from typing import Final
@@ -15,12 +16,14 @@ from utils import get_file_name_with_no_tags as get_search_term
 from logger.logger import log
 from utils.cache import cache
 from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task

 MAIN_GAME_CATEGORY: Final = 0
 EXPANDED_GAME_CATEGORY: Final = 10
 N_SCREENSHOTS: Final = 5
 PS2_IGDB_ID: Final = 8
 SWITCH_IGDB_ID: Final = 130
+ARCADE_IGDB_ID: Final = 52

 PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
 PS2_OPL_INDEX_FILE: Final = os.path.join(
@@ -32,6 +35,8 @@ SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
    os.path.dirname(__file__), "fixtures", "switch_titledb.json"
 )

+MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
+

 class IGDBHandler:
    def __init__(self) -> None:
@@ -183,6 +188,27 @@ class IGDBHandler:
                if index_entry:
                    search_term = index_entry["name"]  # type: ignore

+        if p_igdb_id == ARCADE_IGDB_ID:
+            mame_index = {}
+
+            try:
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            except FileNotFoundError:
+                log.warning("Fetching the MAME XML file from HyperspinFE...")
+                await update_mame_xml_task.run(force=True)
+
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            finally:
+                index_entry = [
+                    game
+                    for game in mame_index["menu"]["game"]
+                    if game["@name"] == search_term
+                ]
+                if index_entry:
+                    search_term = index_entry[0].get("description", search_term)
+
        res = (
            self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY)
            or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY)
--- a/backend/main.py
+++ b/backend/main.py
@@ -19,6 +19,8 @@ from config import (
    SCHEDULED_RESCAN_CRON,
    ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
    SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+    ENABLE_SCHEDULED_UPDATE_MAME_XML,
+    SCHEDULED_UPDATE_MAME_XML_CRON,
 )
 from endpoints import search, platform, rom, identity, oauth, scan  # noqa
 from handler import dbh
@@ -82,6 +84,8 @@ def heartbeat():
        "SCHEDULED_RESCAN_CRON": SCHEDULED_RESCAN_CRON,
        "ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB": ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,  # noqa
        "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON": SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+        "ENABLE_SCHEDULED_UPDATE_MAME_XML": ENABLE_SCHEDULED_UPDATE_MAME_XML,
+        "SCHEDULED_UPDATE_MAME_XML_CRON": SCHEDULED_UPDATE_MAME_XML_CRON,
    }


--- a/backend/scheduler.py
+++ b/backend/scheduler.py
@@ -5,6 +5,7 @@ from tasks.utils import tasks_scheduler
 from logger.logger import log
 from tasks.scan_library import scan_library_task
 from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task

 if __name__ == "__main__":
    if not ENABLE_EXPERIMENTAL_REDIS:
@@ -13,6 +14,7 @@ if __name__ == "__main__":
    # Initialize the tasks
    scan_library_task.init()
    update_switch_titledb_task.init()
+    update_mame_xml_task.init()

    log.info("Starting scheduler")

--- a/backend/tasks/update_mame_xml.py
+++ b/backend/tasks/update_mame_xml.py
@@ -0,0 +1,28 @@
+import os
+from pathlib import Path
+
+from typing import Final
+from config import (
+    ENABLE_SCHEDULED_UPDATE_MAME_XML,
+    SCHEDULED_UPDATE_MAME_XML_CRON,
+)
+from .utils import RemoteFilePullTask
+
+FIXTURE_FILE_PATH: Final = (
+    Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" / "mame.xml"
+)
+
+
+class UpdateMAMEXMLTask(RemoteFilePullTask):
+    def __init__(self):
+        super().__init__(
+            func="tasks.update_mame_xml.update_mame_xml_task.run",
+            description="mame xml update",
+            enabled=ENABLE_SCHEDULED_UPDATE_MAME_XML,
+            cron_string=SCHEDULED_UPDATE_MAME_XML_CRON,
+            url="https://hyperlist.hyperspin-fe.com/genall.php?system=6",
+            file_path=FIXTURE_FILE_PATH,
+        )
+
+
+update_mame_xml_task = UpdateMAMEXMLTask()
--- a/backend/tasks/update_switch_titledb.py
+++ b/backend/tasks/update_switch_titledb.py
@@ -1,17 +1,14 @@
-import requests
 import os
 from pathlib import Path

-from logger.logger import log
 from typing import Final
 from config import (
    ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
    SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
 )
-from .utils import PeriodicTask
+from .utils import RemoteFilePullTask

-RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json"
-FIXTURE_FILE_PATH = (
+FIXTURE_FILE_PATH: Final = (
    Path(os.path.dirname(__file__)).parent
    / "handler"
    / "fixtures"
@@ -19,34 +16,16 @@ FIXTURE_FILE_PATH = (
 )


-class UpdateSwitchTitleDBTask(PeriodicTask):
+class UpdateSwitchTitleDBTask(RemoteFilePullTask):
    def __init__(self):
        super().__init__(
            func="tasks.update_switch_titledb.update_switch_titledb_task.run",
            description="switch titledb update",
            enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
            cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+            url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json",
+            file_path=FIXTURE_FILE_PATH,
        )

-    async def run(self, force: bool = False):
-        if not ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB and not force:
-            log.info("Scheduled switch titledb update not enabled, unscheduling...")
-            self.unschedule()
-            return
-
-        log.info("Scheduled switch titledb update started...")
-
-        try:
-            response = requests.get(RAW_URL)
-            response.raise_for_status()
-
-            with open(FIXTURE_FILE_PATH, "wb") as fixture:
-                fixture.write(response.content)
-
-            log.info("Scheduled switch titledb update done")
-        except requests.exceptions.RequestException as e:
-            log.error("Scheduled switch titledb update failed", exc_info=True)
-            log.error(e)
-

 update_switch_titledb_task = UpdateSwitchTitleDBTask()
--- a/backend/tasks/utils.py
+++ b/backend/tasks/utils.py
@@ -1,3 +1,4 @@
+import requests
 from rq_scheduler import Scheduler
 from abc import ABC, abstractmethod

@@ -73,3 +74,31 @@ class PeriodicTask(ABC):

        tasks_scheduler.cancel(job)
        log.info(f"{self.description.capitalize()} unscheduled.")
+
+
+class RemoteFilePullTask(PeriodicTask):
+    def __init__(self, *args, url: str, file_path: str, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.url = url
+        self.file_path = file_path
+
+    async def run(self, force: bool = False):
+        if not self.enabled and not force:
+            log.info(f"Scheduled {self.description} not enabled, unscheduling...")
+            self.unschedule()
+            return
+
+        log.info(f"Scheduled {self.description} started...")
+
+        try:
+            response = requests.get(self.url)
+            response.raise_for_status()
+
+            with open(self.file_path, "wb") as fixture:
+                fixture.write(response.content)
+
+            log.info(f"Scheduled {self.description} done")
+        except requests.exceptions.RequestException as e:
+            log.error(f"Scheduled {self.description} failed", exc_info=True)
+            log.error(e)
--- a/env.template
+++ b/env.template
@@ -39,3 +39,5 @@ ENABLE_SCHEDULED_RESCAN=true
 SCHEDULED_RESCAN_CRON=0 3 * * *
 ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
 SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * *
+ENABLE_SCHEDULED_UPDATE_MAME_XML=true
+SCHEDULED_UPDATE_MAME_XML_CRON=0 4 * * *
--- a/frontend/src/views/Settings/General/Settings.vue
+++ b/frontend/src/views/Settings/General/Settings.vue
@@ -46,10 +46,17 @@ onBeforeMount(async () => {
  switchUpdate =
    switchUpdate.charAt(0).toLocaleLowerCase() + switchUpdate.substr(1);

+  let mameUpdate = cronstrue.toString(
+    data.SCHEDULED_UPDATE_MAME_XML_CRON,
+    { verbose: true }
+  );
+  mameUpdate = mameUpdate.charAt(0).toLocaleLowerCase() + mameUpdate.substr(1);
+
  heartbeat.value = {
    ...data,
    SCHEDULED_RESCAN_CRON: rescan,
    SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: switchUpdate,
+    SCHEDULED_UPDATE_MAME_XML_CRON: mameUpdate,
  };
 });
 </script>
@@ -175,6 +182,32 @@ onBeforeMount(async () => {
            </p>
          </div>
        </v-col>
+        <v-col
+          cols="12"
+          md="4"
+          sm="6"
+          :class="{
+            'status-item d-flex': true,
+            disabled: !heartbeat.ENABLE_SCHEDULED_UPDATE_MAME_XML,
+          }"
+        >
+          <v-icon
+            :icon="
+              heartbeat.ENABLE_SCHEDULED_UPDATE_MAME_XML
+                ? 'mdi-clock-check-outline'
+                : 'mdi-clock-remove-outline'
+            "
+          />
+          <div class="ml-3">
+            <v-label class="font-weight-bold">
+              Scheduled MAME XML update
+            </v-label>
+            <p class="mt-1">
+              Updates the Nintedo MAME XML file
+              {{ heartbeat.SCHEDULED_UPDATE_MAME_XML_CRON }}
+            </p>
+          </div>
+        </v-col>
      </v-row>
    </v-card-text>
  </v-card>
--- a/poetry.lock
+++ b/poetry.lock
@@ -557,6 +557,7 @@ files = [
    {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
    {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
    {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
+    {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
    {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
    {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
    {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@@ -565,6 +566,7 @@ files = [
    {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
    {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
    {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
+    {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
    {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
    {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
    {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@@ -594,6 +596,7 @@ files = [
    {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
    {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
    {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
+    {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
    {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
    {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
    {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@@ -602,6 +605,7 @@ files = [
    {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
    {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
    {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
+    {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
    {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
    {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
    {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@@ -2180,6 +2184,17 @@ files = [
    {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"},
 ]

+[[package]]
+name = "xmltodict"
+version = "0.13.0"
+description = "Makes working with XML feel like you are working with JSON"
+optional = false
+python-versions = ">=3.4"
+files = [
+    {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"},
+    {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
+]
+
 [[package]]
 name = "yarl"
 version = "1.9.2"
@@ -2270,4 +2285,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "29a5ef6641ea9b51f2b0fc389713b0316bd9feb8b3d89db341bc71449ccead61"
+content-hash = "2f2317b80e297bea8fde4735a5442778119a8022049a17edc853fceb79af2769"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,6 +45,7 @@ python-multipart = "^0.0.6"
 types-python-jose = "^3.3.4.8"
 types-passlib = "^1.7.7.13"
 watchdog = "^3.0.0"
+xmltodict = "^0.13.0"

 [build-system]
 requires = ["poetry-core"]
--- a/pytest.ini
+++ b/pytest.ini
@@ -13,6 +13,8 @@ env =
   ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true
   ENABLE_SCHEDULED_RESCAN=true
   ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
+   ENABLE_SCHEDULED_UPDATE_MAME_XML=true
   RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5
   SCHEDULED_RESCAN_CRON=0 3 * * *
   SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 3 * * *
+   SCHEDULED_UPDATE_MAME_XML_CRON=0 3 * * *