From 9584ba9a370f7c7ecd4d60e7c111d1140e3ffad2 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sun, 14 Dec 2025 13:18:24 -0500 Subject: [PATCH 1/3] handler can see colocated folders --- backend/handler/metadata/gamelist_handler.py | 36 +++++++- examples/config.example.yml | 96 +++++++++++++------- 2 files changed, 96 insertions(+), 36 deletions(-) diff --git a/backend/handler/metadata/gamelist_handler.py b/backend/handler/metadata/gamelist_handler.py index 30b11d6ec..087674b3a 100644 --- a/backend/handler/metadata/gamelist_handler.py +++ b/backend/handler/metadata/gamelist_handler.py @@ -1,3 +1,4 @@ +import glob import os import uuid from pathlib import Path @@ -164,6 +165,37 @@ def extract_media_from_gamelist_rom( ) gamelist_media["video_url"] = f"file://{str(video_path_path)}" + media_map = { + "image_url": "images", + "box2d_url": "covers", + "box2d_back_url": "backcovers", + "box3d_url": "3dboxes", + "fanart_url": "fanart", + "manual_url": "manuals", + "marquee_url": "marquees", + "miximage_url": "miximages", + "physical_url": "physicalmedia", + "screenshot_url": "screenshots", + "title_screen_url": "titlescreens", + "thumbnail_url": "thumbnails", + "video_url": "videos", + } + + path_elem = game.find("path") + if path_elem is not None and path_elem.text: + rom_basename = os.path.basename(path_elem.text) + for media_type, folder_name in media_map.items(): + if gamelist_media[media_type]: + continue + + search_path = os.path.join( + platform_dir, folder_name, f"{os.path.splitext(rom_basename)[0]}.*" + ) + found_files = glob.glob(search_path) + print("found_files", search_path, found_files) + if found_files: + gamelist_media[media_type] = f"file://{str(found_files[0])}" + return gamelist_media @@ -376,8 +408,10 @@ class GamelistHandler(MetadataHandler): # Choose which cover style to use cover_url = rom_metadata["box2d_url"] or rom_metadata["image_url"] + print("URL_COVER", cover_url) if cover_url: rom_data["url_cover"] = cover_url + print("URL_COVER", cover_url) # Grab the manual manual_url = rom_metadata["manual_url"] @@ -433,7 +467,7 @@ class GamelistHandler(MetadataHandler): if gamelist_metadata: rom_specific_paths = populate_rom_specific_paths(gamelist_metadata, rom) # trunk-ignore(mypy/call-arg) - gamelist_metadata.update(**rom_specific_paths) + gamelist_metadata.update(**rom_specific_paths) # type: ignore matched_rom["gamelist_metadata"] = gamelist_metadata return matched_rom diff --git a/examples/config.example.yml b/examples/config.example.yml index f00d75b7f..b71680b0c 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -2,48 +2,74 @@ # Rename this file to `config.yml`, copy it to a `config` folder, and mount that folder as per the docker-compose.example.yml # Only uncomment the lines you want to use/modify, or add new ones where needed -exclude: - # Exclude platforms to be scanned - platforms: [] # ['my_excluded_platform_1', 'my_excluded_platform_2'] +# exclude: +# # Exclude platforms to be scanned +# platforms: +# - excluded_folder_a +# - excluded_folder_b - # Exclude roms or parts of roms to be scanned - roms: - # Single file games section. - # Will not apply to files that are in sub-folders (multi-disc roms, games with updates, DLC, patches, etc.) - single_file: - # Exclude all files with certain extensions to be scanned - extensions: [] # ['xml', 'txt'] +# # Exclude roms or parts of roms to be scanned +# roms: +# # Single file games section. +# # Will not apply to files that are in sub-folders (multi-disc roms, games with updates, DLC, patches, etc.) +# single_file: +# # Exclude all files with certain extensions to be scanned +# extensions: +# - xml +# - txt - # Exclude matched file names to be scanned. - # Supports unix filename pattern matching - # Can also exclude files by extension - names: [] # ['info.txt', '._*', '*.nfo'] +# # Exclude matched file names to be scanned +# # Supports unix filename pattern matching +# names: +# - 'info.txt' +# - '._*' +# - '*.nfo' - # Multi files games section - # Will apply to files that are in sub-folders (multi-disc roms, games with updates, DLC, patches, etc.) - multi_file: - # Exclude matched 'folder' names to be scanned (RomM identifies folders as multi file games) - names: [] # ['my_multi_file_game', 'DLC'] +# # Multi files games section +# # Will apply to files that are in sub-folders (multi-disc roms, games with updates, DLC, patches, etc.) +# multi_file: +# # Exclude matched 'folder' names to be scanned (RomM identifies folders as multi file games) +# # Common ES-DE media folders are listed below +# names: +# - 3dboxes +# - backcovers +# - covers +# - fanart +# - manuals +# - marquees +# - miximages +# - physicalmedia +# - screenshots +# - titlescreens +# - videos +# - downloaded_media +# - media - # Exclude files within sub-folders. - parts: - # Exclude matched file names to be scanned from multi file roms - # Keep in mind that RomM doesn't scan folders inside multi files games, - # so there is no need to exclude folders from inside of multi files games. - names: [] # ['data.xml', '._*'] # Supports unix filename pattern matching +# # Exclude files within sub-folders. +# parts: +# # Exclude matched file names to be scanned from multi file roms +# # Keep in mind that RomM doesn't scan folders inside multi files games, +# # so there is no need to exclude folders from inside of multi files games. +# names: +# - 'data.xml' +# - '._*' # Supports unix filename pattern matching - # Exclude all files with certain extensions to be scanned from multi file roms - extensions: [] # ['xml', 'txt'] +# # Exclude all files with certain extensions to be scanned from multi file roms +# extensions: +# - xml +# - txt -system: - # Asociate different platform names to your current file system platform names - # [your custom platform folder name]: [RomM platform name] - # In this example if you have a 'gc' folder, RomM will treat it like the 'ngc' folder and if you have a 'psx' folder, RomM will treat it like the 'ps' folder - platforms: {} # { gc: 'ngc', psx: 'ps' } - - # Asociate one platform to it's main version - versions: {} # { naomi: 'arcade' } +# system: +# # Asociate different platform names to your current file system platform names +# # [your custom platform folder name]: [RomM platform name] +# # In this example if you have a 'gc' folder, RomM will treat it like the 'ngc' folder +# platforms: +# - gc: ngc +# - ps1: psx +# # Asociate one platform to it's main version (IGDB only) +# versions: +# - naomi: arcade # The folder name where your roms are located # filesystem: From fea6bb361781199cf5411ebbb535597e7f2118a3 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sun, 14 Dec 2025 13:25:14 -0500 Subject: [PATCH 2/3] validate path when fetching --- backend/handler/metadata/gamelist_handler.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/handler/metadata/gamelist_handler.py b/backend/handler/metadata/gamelist_handler.py index 087674b3a..67eb30960 100644 --- a/backend/handler/metadata/gamelist_handler.py +++ b/backend/handler/metadata/gamelist_handler.py @@ -188,11 +188,12 @@ def extract_media_from_gamelist_rom( if gamelist_media[media_type]: continue - search_path = os.path.join( - platform_dir, folder_name, f"{os.path.splitext(rom_basename)[0]}.*" + search_path = fs_platform_handler.validate_path( + os.path.join( + platform_dir, folder_name, f"{os.path.splitext(rom_basename)[0]}.*" + ) ) - found_files = glob.glob(search_path) - print("found_files", search_path, found_files) + found_files = glob.glob(str(search_path)) if found_files: gamelist_media[media_type] = f"file://{str(found_files[0])}" @@ -408,10 +409,8 @@ class GamelistHandler(MetadataHandler): # Choose which cover style to use cover_url = rom_metadata["box2d_url"] or rom_metadata["image_url"] - print("URL_COVER", cover_url) if cover_url: rom_data["url_cover"] = cover_url - print("URL_COVER", cover_url) # Grab the manual manual_url = rom_metadata["manual_url"] From 9b0311def0bc9d6eb201dbc748c1396a29407a50 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sun, 14 Dec 2025 14:19:12 -0500 Subject: [PATCH 3/3] add type suppressions --- backend/handler/metadata/gamelist_handler.py | 164 +++++++------------ examples/config.example.yml | 6 +- 2 files changed, 61 insertions(+), 109 deletions(-) diff --git a/backend/handler/metadata/gamelist_handler.py b/backend/handler/metadata/gamelist_handler.py index 67eb30960..4045bed2e 100644 --- a/backend/handler/metadata/gamelist_handler.py +++ b/backend/handler/metadata/gamelist_handler.py @@ -2,7 +2,7 @@ import glob import os import uuid from pathlib import Path -from typing import NotRequired, TypedDict +from typing import Final, NotRequired, TypedDict from xml.etree.ElementTree import Element # trunk-ignore(bandit/B405) import pydash @@ -64,6 +64,47 @@ class GamelistRom(BaseRom): gamelist_metadata: NotRequired[GamelistMetadata] +ESDE_MEDIA_MAP: Final = { + "image_url": "images", + "box2d_url": "covers", + "box2d_back_url": "backcovers", + "box3d_url": "3dboxes", + "fanart_url": "fanart", + "manual_url": "manuals", + "marquee_url": "marquees", + "miximage_url": "miximages", + "physical_url": "physicalmedia", + "screenshot_url": "screenshots", + "title_screen_url": "titlescreens", + "thumbnail_url": "thumbnails", + "video_url": "videos", +} + +XML_TAG_MAP: Final = { + "image_url": "image", + "box2d_url": "cover", + "box2d_back_url": "backcover", + "box3d_url": "box3d", + "fanart_url": "fanart", + "manual_url": "manual", + "marquee_url": "marquee", + "miximage_url": "miximage", + "physical_url": "physicalmedia", + "screenshot_url": "screenshot", + "title_screen_url": "title_screen", + "thumbnail_url": "thumbnail", + "video_url": "video", +} + + +def _make_file_uri(platform_dir: str, raw_text: str) -> str: + cleaned_text = raw_text.replace("./", "") + validated_path = fs_platform_handler.validate_path( + os.path.join(platform_dir, cleaned_text) + ) + return f"file://{str(validated_path)}" + + def extract_media_from_gamelist_rom( game: Element, platform: Platform ) -> GamelistMetadataMedia: @@ -85,117 +126,29 @@ def extract_media_from_gamelist_rom( video_url=None, ) - image_elem = game.find("image") - video_elem = game.find("video") - box3d_elem = game.find("box3d") - box2d_back_elem = game.find("backcover") - box2d_elem = game.find("cover") - fanart_elem = game.find("fanart") - manual_elem = game.find("manual") - marquee_elem = game.find("marquee") - miximage_elem = game.find("miximage") - physical_elem = game.find("physicalmedia") - screenshot_elem = game.find("screenshot") - title_screen_elem = game.find("title_screen") - thumbnail_elem = game.find("thumbnail") - - if image_elem is not None and image_elem.text: - image_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{image_elem.text.replace('./', '')}" - ) - gamelist_media["image_url"] = f"file://{str(image_path_path)}" - if box2d_elem is not None and box2d_elem.text: - box2d_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{box2d_elem.text.replace('./', '')}" - ) - gamelist_media["box2d_url"] = f"file://{str(box2d_path_path)}" - if box2d_back_elem is not None and box2d_back_elem.text: - box2d_back_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{box2d_back_elem.text.replace('./', '')}" - ) - gamelist_media["box2d_back_url"] = f"file://{str(box2d_back_path_path)}" - if box3d_elem is not None and box3d_elem.text: - box3d_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{box3d_elem.text.replace('./', '')}" - ) - gamelist_media["box3d_url"] = f"file://{str(box3d_path_path)}" - if fanart_elem is not None and fanart_elem.text: - fanart_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{fanart_elem.text.replace('./', '')}" - ) - gamelist_media["fanart_url"] = f"file://{str(fanart_path_path)}" - if manual_elem is not None and manual_elem.text: - manual_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{manual_elem.text.replace('./', '')}" - ) - gamelist_media["manual_url"] = f"file://{str(manual_path_path)}" - if marquee_elem is not None and marquee_elem.text: - marquee_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{marquee_elem.text.replace('./', '')}" - ) - gamelist_media["marquee_url"] = f"file://{str(marquee_path_path)}" - if miximage_elem is not None and miximage_elem.text: - miximage_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{miximage_elem.text.replace('./', '')}" - ) - gamelist_media["miximage_url"] = f"file://{str(miximage_path_path)}" - if physical_elem is not None and physical_elem.text: - physical_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{physical_elem.text.replace('./', '')}" - ) - gamelist_media["physical_url"] = f"file://{str(physical_path_path)}" - if screenshot_elem is not None and screenshot_elem.text: - screenshot_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{screenshot_elem.text.replace('./', '')}" - ) - gamelist_media["screenshot_url"] = f"file://{str(screenshot_path_path)}" - if title_screen_elem is not None and title_screen_elem.text: - title_screen_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{title_screen_elem.text.replace('./', '')}" - ) - gamelist_media["title_screen_url"] = f"file://{str(title_screen_path_path)}" - if thumbnail_elem is not None and thumbnail_elem.text: - thumbnail_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{thumbnail_elem.text.replace('./', '')}" - ) - gamelist_media["thumbnail_url"] = f"file://{str(thumbnail_path_path)}" - if video_elem is not None and video_elem.text: - video_path_path = fs_platform_handler.validate_path( - f"{platform_dir}/{video_elem.text.replace('./', '')}" - ) - gamelist_media["video_url"] = f"file://{str(video_path_path)}" - - media_map = { - "image_url": "images", - "box2d_url": "covers", - "box2d_back_url": "backcovers", - "box3d_url": "3dboxes", - "fanart_url": "fanart", - "manual_url": "manuals", - "marquee_url": "marquees", - "miximage_url": "miximages", - "physical_url": "physicalmedia", - "screenshot_url": "screenshots", - "title_screen_url": "titlescreens", - "thumbnail_url": "thumbnails", - "video_url": "videos", - } + # Check explicit XML elements defined in gamelist.xml + for media_key, xml_tag in XML_TAG_MAP.items(): + elem = game.find(xml_tag) + if elem is not None and elem.text: + # trunk-ignore(mypy/literal-required) + gamelist_media[media_key] = _make_file_uri(platform_dir, elem.text) + # Fallback to searching media folders by ROM basename path_elem = game.find("path") if path_elem is not None and path_elem.text: - rom_basename = os.path.basename(path_elem.text) - for media_type, folder_name in media_map.items(): - if gamelist_media[media_type]: + rom_stem = os.path.splitext(os.path.basename(path_elem.text))[0] + + for media_key, folder_name in ESDE_MEDIA_MAP.items(): + # trunk-ignore(mypy/literal-required) + if gamelist_media[media_key]: continue - search_path = fs_platform_handler.validate_path( - os.path.join( - platform_dir, folder_name, f"{os.path.splitext(rom_basename)[0]}.*" - ) - ) + search_pattern = os.path.join(platform_dir, folder_name, f"{rom_stem}.*") + search_path = fs_platform_handler.validate_path(search_pattern) found_files = glob.glob(str(search_path)) if found_files: - gamelist_media[media_type] = f"file://{str(found_files[0])}" + # trunk-ignore(mypy/literal-required) + gamelist_media[media_key] = f"file://{str(found_files[0])}" return gamelist_media @@ -465,7 +418,6 @@ class GamelistHandler(MetadataHandler): # Populate ROM-specific paths using the actual rom object if gamelist_metadata: rom_specific_paths = populate_rom_specific_paths(gamelist_metadata, rom) - # trunk-ignore(mypy/call-arg) gamelist_metadata.update(**rom_specific_paths) # type: ignore matched_rom["gamelist_metadata"] = gamelist_metadata diff --git a/examples/config.example.yml b/examples/config.example.yml index b71680b0c..0d00a8b47 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -64,12 +64,12 @@ # # [your custom platform folder name]: [RomM platform name] # # In this example if you have a 'gc' folder, RomM will treat it like the 'ngc' folder # platforms: -# - gc: ngc -# - ps1: psx +# gc: ngc +# ps1: psx # # Asociate one platform to it's main version (IGDB only) # versions: -# - naomi: arcade +# naomi: arcade # The folder name where your roms are located # filesystem: