rommapp/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-02-18 00:27:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Cache parsed gamelist.xml files on each scan

Browse Source
This commit is contained in:
Georges-Antoine Assi
2025-11-19 18:00:12 -05:00
parent 04b2439d47
commit 8fe435e0b4
3 changed files with 59 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/endpoints/sockets/scan.py
View File

@@ -27,6 +27,7 @@ from handler.filesystem import (
fs_rom_handler,
)
from handler.filesystem.roms_handler import FSRom
from handler.metadata import meta_gamelist_handler
from handler.metadata.ss_handler import get_preferred_media_types
from handler.redis_handler import get_job_func_name, high_prio_queue, redis_client
from handler.scan_handler import (
@@ -276,9 +277,13 @@ async def _identify_rom(
)
if should_update_files:
log.debug(f"Calculating file hashes for {rom.fs_name}...")
rom_files, rom_crc_c, rom_md5_h, rom_sha1_h, rom_ra_h = (
await fs_rom_handler.get_rom_files(rom)
)
(
rom_files,
rom_crc_c,
rom_md5_h,
rom_sha1_h,
rom_ra_h,
) = await fs_rom_handler.get_rom_files(rom)
fs_rom.update(
{
"files": rom_files,
@@ -583,6 +588,9 @@ async def scan_platforms(
await socket_manager.emit("scan:done_ko", e.message)
return scan_stats
# Clear the gamelist cache to ensure we're using fresh gamelist.xml data
meta_gamelist_handler.clear_cache()
# Precalculate total platforms and ROMs
total_roms = 0
for platform_slug in fs_platforms:

61
backend/handler/metadata/gamelist_handler.py
View File

@@ -106,22 +106,22 @@ def extract_media_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetadata
if image_elem is not None and image_elem.text:
image_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{image_elem.text.replace("./", "")}"
f"{platform_dir}/{image_elem.text.replace('./', '')}"
)
gamelist_media["image_url"] = f"file://{str(image_path_path)}"
if box2d_elem is not None and box2d_elem.text:
box2d_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{box2d_elem.text.replace("./", "")}"
f"{platform_dir}/{box2d_elem.text.replace('./', '')}"
)
gamelist_media["box2d_url"] = f"file://{str(box2d_path_path)}"
if box2d_back_elem is not None and box2d_back_elem.text:
box2d_back_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{box2d_back_elem.text.replace("./", "")}"
f"{platform_dir}/{box2d_back_elem.text.replace('./', '')}"
)
gamelist_media["box2d_back_url"] = f"file://{str(box2d_back_path_path)}"
if box3d_elem is not None and box3d_elem.text:
box3d_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{box3d_elem.text.replace("./", "")}"
f"{platform_dir}/{box3d_elem.text.replace('./', '')}"
)
gamelist_media["box3d_url"] = f"file://{str(box3d_path_path)}"
@@ -131,17 +131,17 @@ def extract_media_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetadata
)
if fanart_elem is not None and fanart_elem.text:
fanart_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{fanart_elem.text.replace("./", "")}"
f"{platform_dir}/{fanart_elem.text.replace('./', '')}"
)
gamelist_media["fanart_url"] = f"file://{str(fanart_path_path)}"
if manual_elem is not None and manual_elem.text:
manual_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{manual_elem.text.replace("./", "")}"
f"{platform_dir}/{manual_elem.text.replace('./', '')}"
)
gamelist_media["manual_url"] = f"file://{str(manual_path_path)}"
if marquee_elem is not None and marquee_elem.text:
marquee_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{marquee_elem.text.replace("./", "")}"
f"{platform_dir}/{marquee_elem.text.replace('./', '')}"
)
gamelist_media["marquee_url"] = f"file://{str(marquee_path_path)}"
@@ -151,7 +151,7 @@ def extract_media_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetadata
)
if miximage_elem is not None and miximage_elem.text:
miximage_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{miximage_elem.text.replace("./", "")}"
f"{platform_dir}/{miximage_elem.text.replace('./', '')}"
)
gamelist_media["miximage_url"] = f"file://{str(miximage_path_path)}"
@@ -161,7 +161,7 @@ def extract_media_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetadata
)
if physical_elem is not None and physical_elem.text:
physical_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{physical_elem.text.replace("./", "")}"
f"{platform_dir}/{physical_elem.text.replace('./', '')}"
)
gamelist_media["physical_url"] = f"file://{str(physical_path_path)}"
@@ -171,22 +171,22 @@ def extract_media_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetadata
)
if screenshot_elem is not None and screenshot_elem.text:
screenshot_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{screenshot_elem.text.replace("./", "")}"
f"{platform_dir}/{screenshot_elem.text.replace('./', '')}"
)
gamelist_media["screenshot_url"] = f"file://{str(screenshot_path_path)}"
if title_screen_elem is not None and title_screen_elem.text:
title_screen_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{title_screen_elem.text.replace("./", "")}"
f"{platform_dir}/{title_screen_elem.text.replace('./', '')}"
)
gamelist_media["title_screen_url"] = f"file://{str(title_screen_path_path)}"
if thumbnail_elem is not None and thumbnail_elem.text:
thumbnail_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{thumbnail_elem.text.replace("./", "")}"
f"{platform_dir}/{thumbnail_elem.text.replace('./', '')}"
)
gamelist_media["thumbnail_url"] = f"file://{str(thumbnail_path_path)}"
if video_elem is not None and video_elem.text:
video_path_path = fs_platform_handler.validate_path(
f"{platform_dir}/{video_elem.text.replace("./", "")}"
f"{platform_dir}/{video_elem.text.replace('./', '')}"
)
gamelist_media["video_url"] = f"file://{str(video_path_path)}"
@@ -250,6 +250,24 @@ def extract_metadata_from_gamelist_rom(rom: Rom, game: Element) -> GamelistMetad
class GamelistHandler(MetadataHandler):
"""Handler for ES-DE gamelist.xml metadata source"""
def __init__(self):
# Cache for storing parsed gamelist data by platform ID
self._gamelist_cache = {}
def clear_cache(self):
"""Clear the gamelist cache"""
self._gamelist_cache.clear()
def invalidate_cache_for_platform(self, platform_id: int):
"""Invalidate cached data for a specific platform"""
keys_to_remove = [
key
for key in self._gamelist_cache.keys()
if key.startswith(f"{platform_id}_")
]
for key in keys_to_remove:
del self._gamelist_cache[key]
@classmethod
def is_enabled(cls) -> bool:
return True
@@ -271,7 +289,19 @@ class GamelistHandler(MetadataHandler):
def _parse_gamelist_xml(
self, gamelist_path: Path, platform: Platform, rom: Rom
) -> dict[str, GamelistRom]:
"""Parse a gamelist.xml file and return ROM data indexed by filename"""
"""Parse a gamelist.xml file and return ROM data indexed by filename.
Results are cached by platform ID and gamelist path to avoid
re-parsing the same file multiple times.
"""
# Check if we already have cached data for this platform and gamelist path
platform_id = platform.id
cache_key = f"{platform_id}_{str(gamelist_path)}"
if cache_key in self._gamelist_cache:
log.debug(f"Using cached gamelist data for platform {platform_id}")
return self._gamelist_cache[cache_key]
preferred_media_types = get_preferred_media_types()
roms_data: dict[str, GamelistRom] = {}
@@ -352,6 +382,9 @@ class GamelistHandler(MetadataHandler):
# Store by filename for matching
roms_data[rom_filename] = rom_data
# Cache the parsed data for this platform and gamelist path
self._gamelist_cache[cache_key] = roms_data
except ET.ParseError as e:
log.warning(f"Failed to parse gamelist.xml at {gamelist_path}: {e}")
except Exception as e:

2
backend/handler/scan_handler.py
View File

@@ -447,7 +447,7 @@ async def scan_rom(
if playmatch_rom["igdb_id"] is not None:
log.debug(
f"{hl(rom_attrs['fs_name'])} identified by Playmatch as "
f"{hl(str(playmatch_rom["igdb_id"]), color=BLUE)} {emoji.EMOJI_ALIEN_MONSTER}",
f"{hl(str(playmatch_rom['igdb_id']), color=BLUE)} {emoji.EMOJI_ALIEN_MONSTER}",
extra=LOGGER_MODULE_NAME,
)