Rebuild moby api on new code

This commit is contained in:
Georges-Antoine Assi
2024-02-13 18:11:21 -05:00
parent 9845d6f36c
commit ce9deb4f3c
33 changed files with 1054 additions and 307 deletions

View File

@@ -0,0 +1,40 @@
"""empty message
Revision ID: 0015_mobygames_data
Revises: 0014_asset_files
Create Date: 2024-02-13 17:57:25.936825
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = "0015_mobygames_data"
down_revision = "0014_asset_files"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.add_column(sa.Column("moby_id", sa.Integer(), nullable=True))
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.add_column(sa.Column("moby_id", sa.Integer(), nullable=True))
batch_op.add_column(sa.Column("moby_metadata", mysql.JSON(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table("roms", schema=None) as batch_op:
batch_op.drop_column("moby_metadata")
batch_op.drop_column("moby_id")
with op.batch_alter_table("platforms", schema=None) as batch_op:
batch_op.drop_column("moby_id")
# ### end Alembic commands ###

View File

@@ -41,6 +41,9 @@ IGDB_CLIENT_SECRET: Final = os.environ.get(
# STEAMGRIDDB
STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "")
# MOBYGAMES
MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "")
# DB DRIVERS
ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")

View File

@@ -9,6 +9,7 @@ class PlatformSchema(BaseModel):
fs_slug: str
igdb_id: Optional[int] = None
sgdb_id: Optional[int] = None
moby_id: Optional[int] = None
name: Optional[str]
logo_path: str
rom_count: int

View File

@@ -5,7 +5,7 @@ from endpoints.responses.assets import SaveSchema, ScreenshotSchema, StateSchema
from fastapi import Request
from fastapi.responses import StreamingResponse
from handler import socket_handler
from handler.igdb_handler import IGDBRelatedGame
from handler.metadata_handler.igdb_handler import IGDBRelatedGame
from pydantic import BaseModel, computed_field, Field
from models.rom import Rom
from typing_extensions import TypedDict, NotRequired
@@ -28,6 +28,7 @@ class RomSchema(BaseModel):
id: int
igdb_id: Optional[int]
sgdb_id: Optional[int]
moby_id: Optional[int]
platform_id: int
platform_slug: str

View File

@@ -1,5 +1,10 @@
from handler.igdb_handler import IGDBRom
from pydantic import BaseModel
class SearchRomSchema(IGDBRom):
pass
class SearchRomSchema(BaseModel):
igdb_id: int | None = None
moby_id: int | None = None
slug: str
name: str
summary: str
url_cover: str
url_screenshots: list[str]

View File

@@ -3,7 +3,7 @@ from handler.scan_handler import _get_main_platform_igdb_id
from decorators.auth import protected_route
from endpoints.responses.search import SearchRomSchema
from fastapi import APIRouter, Request, HTTPException, status
from handler import db_rom_handler, igdb_handler
from handler import db_rom_handler, igdb_handler, moby_handler
from logger.logger import log
router = APIRouter()
@@ -31,6 +31,9 @@ async def search_rom(
"""
rom = db_rom_handler.get_roms(rom_id)
if not rom:
return []
search_term = search_term or rom.file_name_no_tags
log.info(emoji.emojize(":magnifying_glass_tilted_right: IGDB Searching"))
@@ -40,7 +43,8 @@ async def search_rom(
log.info(emoji.emojize(f":video_game: {rom.platform_slug}: {rom.file_name}"))
if search_by.lower() == "id":
try:
matched_roms = igdb_handler.get_matched_roms_by_id(int(search_term))
igdb_matched_roms = igdb_handler.get_matched_roms_by_id(int(search_term))
moby_matched_roms = moby_handler.get_matched_roms_by_id(int(search_term))
except ValueError:
log.error(f"Search error: invalid ID '{search_term}'")
raise HTTPException(
@@ -48,14 +52,33 @@ async def search_rom(
detail=f"Search error: invalid ID '{search_term}'",
)
elif search_by.lower() == "name":
matched_roms = igdb_handler.get_matched_roms_by_name(
igdb_matched_roms = igdb_handler.get_matched_roms_by_name(
search_term, _get_main_platform_igdb_id(rom.platform), search_extended
)
moby_matched_roms = moby_handler.get_matched_roms_by_name(
search_term, rom.platform.moby_id
)
merged_dict = {item["name"]: item for item in igdb_matched_roms}
for item in moby_matched_roms:
merged_dict[item["name"]] = {**item, **merged_dict.get(item["name"], {})}
matched_roms = [
{
**{
"slug": "",
"name": "",
"summary": "",
"url_cover": "",
"url_screenshots": [],
},
**item,
}
for item in list(merged_dict.values())
]
log.info("Results:")
results = []
for m_rom in matched_roms:
log.info(f"\t - {m_rom['name']}")
results.append(m_rom)
return results
return matched_roms

View File

@@ -96,7 +96,7 @@ async def scan_platforms(
not rom
or rom.id in selected_roms
or complete_rescan
or (rescan_unidentified and not rom.igdb_id)
or (rescan_unidentified and not rom.igdb_id and not rom.moby_id)
):
scanned_rom = await scan_rom(platform, fs_rom)
if rom:
@@ -115,9 +115,7 @@ async def scan_platforms(
},
)
db_rom_handler.purge_roms(
platform.id, [rom["file_name"] for rom in fs_roms]
)
db_rom_handler.purge_roms(platform.id, [rom["file_name"] for rom in fs_roms])
db_platform_handler.purge_platforms(fs_platforms)
log.info(emoji.emojize(":check_mark: Scan completed "))

View File

@@ -11,12 +11,14 @@ from handler.fs_handler.fs_platforms_handler import FSPlatformsHandler
from handler.fs_handler.fs_resources_handler import FSResourceHandler
from handler.fs_handler.fs_roms_handler import FSRomsHandler
from handler.gh_handler import GHHandler
from handler.igdb_handler import IGDBHandler
from handler.sgdb_handler import SGDBHandler
from handler.metadata_handler.igdb_handler import IGDBHandler
from handler.metadata_handler.moby_handler import MobyGamesHandler
from handler.metadata_handler.sgdb_handler import SGDBHandler
from handler.socket_handler import SocketHandler
igdb_handler = IGDBHandler()
sgdb_handler = SGDBHandler()
moby_handler = MobyGamesHandler()
github_handler = GHHandler()
auth_handler = AuthHandler()
oauth_handler = OAuthHandler()

View File

@@ -1,12 +1,12 @@
from decorators.database import begin_session
from handler.db_handler import DBHandler
from models.rom import Rom
from sqlalchemy import and_, delete, func, select, update, or_
from sqlalchemy import and_, delete, func, select, update, or_, Select
from sqlalchemy.orm import Session
class DBRomsHandler(DBHandler):
def _filter(self, data, platform_id, search_term):
def _filter(data: Select[Rom], platform_id: int | None, search_term: str):
if platform_id:
data = data.filter_by(platform_id=platform_id)
@@ -20,7 +20,7 @@ class DBRomsHandler(DBHandler):
return data
def _order(self, data, order_by, order_dir):
def _order(data: Select[Rom], order_by: str, order_dir: str):
if order_by == "id":
_column = Rom.id
else:

View File

@@ -0,0 +1,130 @@
import json
import xmltodict
import os
import re
from typing import Final
from logger.logger import log
from tasks.update_mame_xml import update_mame_xml_task
from tasks.update_switch_titledb import update_switch_titledb_task
PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
PS2_OPL_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "ps2_opl_index.json"
)
SWITCH_TITLEDB_REGEX: Final = r"(70[0-9]{12})"
SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_titledb.json"
)
SWITCH_PRODUCT_ID_REGEX: Final = r"(0100[0-9A-F]{12})"
SWITCH_PRODUCT_ID_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_product_ids.json"
)
MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
class MetadataHandler:
@staticmethod
def normalize_search_term(search_term: str) -> str:
return (
search_term.replace("\u2122", "") # Remove trademark symbol
.replace("\u00ae", "") # Remove registered symbol
.replace("\u00a9", "") # Remove copywrite symbol
.replace("\u2120", "") # Remove service mark symbol
.strip() # Remove leading and trailing spaces
)
@staticmethod
def _normalize_cover_url(url: str) -> str:
return f"https:{url.replace('https:', '')}"
async def _ps2_opl_format(self, match: re.Match[str], search_term: str) -> str:
serial_code = match.group(1)
with open(PS2_OPL_INDEX_FILE, "r") as index_json:
opl_index = json.loads(index_json.read())
index_entry = opl_index.get(serial_code, None)
if index_entry:
search_term = index_entry["Name"] # type: ignore
return search_term
async def _switch_titledb_format(self, match: re.Match[str], search_term: str) -> str:
titledb_index = {}
title_id = match.group(1)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = titledb_index.get(title_id, None)
if index_entry:
search_term = index_entry["name"] # type: ignore
return search_term
async def _switch_productid_format(self, match: re.Match[str], search_term: str) -> str:
product_id_index = {}
product_id = match.group(1)
# Game updates have the same product ID as the main application, except with bitmask 0x800 set
product_id = list(product_id)
product_id[-3] = "0"
product_id = "".join(product_id)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = product_id_index.get(product_id, None)
if index_entry:
search_term = index_entry["name"] # type: ignore
return search_term
async def _mame_format(self, search_term: str) -> str:
from handler import fs_rom_handler
mame_index = {"menu": {"game": []}}
try:
with open(MAME_XML_FILE, "r") as index_xml:
mame_index = xmltodict.parse(index_xml.read())
except FileNotFoundError:
log.warning("Fetching the MAME XML file from HyperspinFE...")
await update_mame_xml_task.run(force=True)
try:
with open(MAME_XML_FILE, "r") as index_xml:
mame_index = xmltodict.parse(index_xml.read())
except FileNotFoundError:
log.error("Could not fetch the MAME XML file from HyperspinFE")
finally:
index_entry = [
game
for game in mame_index["menu"]["game"]
if game["@name"] == search_term
]
if index_entry:
search_term = fs_rom_handler.get_file_name_with_no_tags(
index_entry[0].get("description", search_term)
)
return search_term

View File

@@ -1,23 +1,24 @@
import functools
import json
import os
import re
import sys
import time
from typing import Final, Optional
from typing import Final, Optional, TypedDict, NotRequired
import pydash
import requests
import xmltodict
from config import IGDB_CLIENT_ID, IGDB_CLIENT_SECRET
from handler.redis_handler import cache
from logger.logger import log
from requests.exceptions import HTTPError, Timeout
from tasks.update_mame_xml import update_mame_xml_task
from tasks.update_switch_titledb import update_switch_titledb_task
from typing_extensions import TypedDict
from unidecode import unidecode as uc
from . import (
MetadataHandler,
PS2_OPL_REGEX,
SWITCH_TITLEDB_REGEX,
SWITCH_PRODUCT_ID_REGEX,
)
MAIN_GAME_CATEGORY: Final = 0
EXPANDED_GAME_CATEGORY: Final = 10
N_SCREENSHOTS: Final = 5
@@ -25,27 +26,11 @@ PS2_IGDB_ID: Final = 8
SWITCH_IGDB_ID: Final = 130
ARCADE_IGDB_IDS: Final = [52, 79, 80]
PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
PS2_OPL_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "ps2_opl_index.json"
)
SWITCH_TITLEDB_REGEX: Final = r"(70[0-9]{12})"
SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_titledb.json"
)
SWITCH_PRODUCT_ID_REGEX: Final = r"(0100[0-9A-F]{12})"
SWITCH_PRODUCT_ID_FILE: Final = os.path.join(
os.path.dirname(__file__), "fixtures", "switch_product_ids.json"
)
MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
class IGDBPlatform(TypedDict):
igdb_id: int
name: str
slug: str
name: NotRequired[str]
class IGDBRelatedGame(TypedDict):
@@ -77,12 +62,12 @@ class IGDBMetadata(TypedDict):
class IGDBRom(TypedDict):
igdb_id: Optional[int]
name: Optional[str]
slug: Optional[str]
summary: Optional[str]
url_cover: str
url_screenshots: list[str]
igdb_id: int | None
slug: NotRequired[str]
name: NotRequired[str]
summary: NotRequired[str]
url_cover: NotRequired[str]
url_screenshots: NotRequired[list[str]]
igdb_metadata: Optional[IGDBMetadata]
@@ -137,7 +122,7 @@ def extract_metadata_from_igdb_rom(rom: dict) -> IGDBMetadata:
)
class IGDBHandler:
class IGDBHandler(MetadataHandler):
def __init__(self) -> None:
self.platform_endpoint = "https://api.igdb.com/v4/platforms"
self.platform_version_endpoint = "https://api.igdb.com/v4/platform_versions"
@@ -204,23 +189,9 @@ class IGDBHandler:
return res.json()
@staticmethod
def _normalize_search_term(search_term: str) -> str:
return (
search_term.replace("\u2122", "") # Remove trademark symbol
.replace("\u00ae", "") # Remove registered symbol
.replace("\u00a9", "") # Remove copywrite symbol
.replace("\u2120", "") # Remove service mark symbol
.strip() # Remove leading and trailing spaces
)
@staticmethod
def _normalize_cover_url(url: str) -> str:
return f"https:{url.replace('https:', '')}" if url != "" else ""
def _search_rom(
self, search_term: str, platform_idgb_id: int, category: int = 0
) -> dict:
) -> dict | None:
search_term = uc(search_term)
category_filter: str = f"& category={category}" if category else ""
roms = self._request(
@@ -246,95 +217,7 @@ class IGDBHandler:
or rom["slug"].lower() == search_term.lower()
]
return pydash.get(exact_matches or roms, "[0]", {})
async def _ps2_opl_format(self, match: re.Match[str], search_term: str) -> str:
serial_code = match.group(1)
with open(PS2_OPL_INDEX_FILE, "r") as index_json:
opl_index = json.loads(index_json.read())
index_entry = opl_index.get(serial_code, None)
if index_entry:
search_term = index_entry["Name"] # type: ignore
return search_term
async def _switch_titledb_format(self, match: re.Match[str], search_term: str) -> str:
titledb_index = {}
title_id = match.group(1)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json:
titledb_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = titledb_index.get(title_id, None)
if index_entry:
search_term = index_entry["name"] # type: ignore
return search_term
async def _switch_productid_format(self, match: re.Match[str], search_term: str) -> str:
product_id_index = {}
product_id = match.group(1)
# Game updates have the same product ID as the main application, except with bitmask 0x800 set
product_id = list(product_id)
product_id[-3] = "0"
product_id = "".join(product_id)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.warning("Fetching the Switch titleDB index file...")
await update_switch_titledb_task.run(force=True)
try:
with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json:
product_id_index = json.loads(index_json.read())
except FileNotFoundError:
log.error("Could not fetch the Switch titleDB index file")
finally:
index_entry = product_id_index.get(product_id, None)
if index_entry:
search_term = index_entry["name"] # type: ignore
return search_term
async def _mame_format(self, search_term: str) -> str:
from handler import fs_rom_handler
mame_index = {"menu": {"game": []}}
try:
with open(MAME_XML_FILE, "r") as index_xml:
mame_index = xmltodict.parse(index_xml.read())
except FileNotFoundError:
log.warning("Fetching the MAME XML file from HyperspinFE...")
await update_mame_xml_task.run(force=True)
try:
with open(MAME_XML_FILE, "r") as index_xml:
mame_index = xmltodict.parse(index_xml.read())
except FileNotFoundError:
log.error("Could not fetch the MAME XML file from HyperspinFE")
finally:
index_entry = [
game
for game in mame_index["menu"]["game"]
if game["@name"] == search_term
]
if index_entry:
search_term = fs_rom_handler.get_file_name_with_no_tags(
index_entry[0].get("description", search_term)
)
return search_term
return pydash.get(exact_matches or roms, "[0]", None)
@check_twitch_token
def get_platform(self, slug: str) -> IGDBPlatform:
@@ -344,26 +227,27 @@ class IGDBHandler:
)
platform = pydash.get(platforms, "[0]", None)
if platform:
return IGDBPlatform(
igdb_id=platform["id"],
slug=slug,
name=platform["name"],
)
# Check if platform is a version if not found
if not platform:
platform_versions = self._request(
self.platform_version_endpoint,
data=f'fields {",".join(self.platforms_fields)}; where slug="{slug.lower()}";',
)
version = pydash.get(platform_versions, "[0]", None)
if not version:
return IGDBPlatform(igdb_id=None, name=slug.replace("-", " ").title())
return IGDBPlatform(
igdb_id=version.get("id", None),
name=version.get("name", slug),
)
return IGDBPlatform(
igdb_id=platform.get("id", None),
name=platform.get("name", slug),
platform_versions = self._request(
self.platform_version_endpoint,
data=f'fields {",".join(self.platforms_fields)}; where slug="{slug.lower()}";',
)
version = pydash.get(platform_versions, "[0]", None)
if version:
return IGDBPlatform(
igdb_id=version["id"],
slug=slug,
name=version["name"],
)
return IGDBPlatform(igdb_id=None, slug=slug)
@check_twitch_token
async def get_rom(self, file_name: str, platform_idgb_id: int) -> IGDBRom:
@@ -398,10 +282,13 @@ class IGDBHandler:
or self._search_rom(search_term, platform_idgb_id)
)
if not rom:
return IGDBRom(igdb_id=None)
return IGDBRom(
igdb_id=rom.get("id", None),
name=rom.get("name", search_term),
slug=rom.get("slug", ""),
igdb_id=rom["id"],
slug=rom["slug"],
name=rom["name"],
summary=rom.get("summary", ""),
url_cover=self._normalize_cover_url(rom.get("cover", {}).get("url", "")),
url_screenshots=[
@@ -419,12 +306,15 @@ class IGDBHandler:
self.games_endpoint,
f'fields {",".join(self.games_fields)}; where id={igdb_id};',
)
rom = pydash.get(roms, "[0]", {})
rom = pydash.get(roms, "[0]", None)
if not rom:
return IGDBRom(igdb_id=None)
return IGDBRom(
igdb_id=igdb_id,
name=rom.get("name", ""),
slug=rom.get("slug", ""),
igdb_id=rom["id"],
slug=rom["slug"],
name=rom["name"],
summary=rom.get("summary", ""),
url_cover=self._normalize_cover_url(rom.get("cover", {}).get("url", "")),
url_screenshots=[
@@ -438,13 +328,7 @@ class IGDBHandler:
@check_twitch_token
def get_matched_roms_by_id(self, igdb_id: int) -> list[IGDBRom]:
matched_rom = self.get_rom_by_id(igdb_id)
matched_rom.update(
url_cover=matched_rom.get("url_cover", "").replace(
"t_thumb", "t_cover_big"
),
)
return [matched_rom]
return [self.get_rom_by_id(igdb_id)]
@check_twitch_token
def get_matched_roms_by_name(
@@ -504,22 +388,28 @@ class IGDBHandler:
return [
IGDBRom(
igdb_id=rom.get("id"),
name=rom.get("name", search_term),
slug=rom.get("slug", ""),
summary=rom.get("summary", ""),
url_cover=self._normalize_cover_url(
rom.get("cover", {})
.get("url", "")
.replace("t_thumb", "t_cover_big")
),
url_screenshots=[
self._normalize_cover_url(s.get("url", "")).replace(
"t_thumb", "t_original"
)
for s in rom.get("screenshots", [])
],
igdb_metadata=extract_metadata_from_igdb_rom(rom),
{
k: v
for k, v in {
"igdb_id": rom["id"],
"slug": rom["slug"],
"name": rom["name"],
"summary": rom.get("summary", ""),
"url_cover": self._normalize_cover_url(
rom.get("cover", {})
.get("url", "")
.replace("t_thumb", "t_cover_big")
),
"url_screenshots": [
self._normalize_cover_url(s.get("url", "")).replace(
"t_thumb", "t_original"
)
for s in rom.get("screenshots", [])
],
"igdb_metadata": extract_metadata_from_igdb_rom(rom),
}.items()
if v
}
)
for rom in matched_roms
]

View File

@@ -0,0 +1,615 @@
import pydash
import requests
import yarl
import re
import time
from config import MOBYGAMES_API_KEY
from typing import Final, TypedDict, NotRequired, Optional
from requests.exceptions import HTTPError, Timeout
from logger.logger import log
from unidecode import unidecode as uc
from . import (
MetadataHandler,
PS2_OPL_REGEX,
SWITCH_TITLEDB_REGEX,
SWITCH_PRODUCT_ID_REGEX,
)
PS2_MOBY_ID: Final = 7
SWITCH_MOBY_ID: Final = 203
ARCADE_MOBY_IDS: Final = [143, 36]
class MobyGamesPlatform(TypedDict):
moby_id: int
slug: str
name: NotRequired[str]
class MobyMetadata(TypedDict):
moby_score: str
genres: list[str]
alternate_titles: list[str]
platforms: list[MobyGamesPlatform]
class MobyGamesRom(TypedDict):
moby_id: int | None
slug: NotRequired[str]
name: NotRequired[str]
summary: NotRequired[str]
url_cover: NotRequired[str]
url_screenshots: NotRequired[list[str]]
moby_metadata: Optional[MobyMetadata]
def extract_metadata_from_moby_rom(rom: dict) -> MobyMetadata:
return MobyMetadata(
{
"moby_score": str(rom.get("moby_score", "")),
"genres": rom.get("genres.genre_name", []),
"alternate_titles": rom.get("alternate_titles.title", []),
"platforms": [
{
"moby_id": p["platform_id"],
"slug": MOBY_ID_TO_SLUG[p["platform_id"]],
"name": p["platform_name"],
}
for p in rom.get("platforms", [])
],
}
)
class MobyGamesHandler(MetadataHandler):
def __init__(self) -> None:
self.platform_url = "https://api.mobygames.com/v1/platforms"
self.games_url = "https://api.mobygames.com/v1/games"
def _request(self, url: str, timeout: int = 120) -> dict:
authorized_url = yarl.URL(url).update_query(api_key=MOBYGAMES_API_KEY)
try:
res = requests.get(authorized_url, timeout=timeout)
res.raise_for_status()
return res.json()
except HTTPError as err:
if err.response.status_code != 429:
log.error(err)
return {}
# Retry after 2 seconds if rate limit hit
time.sleep(2)
except Timeout:
# Retry the request once if it times out
pass
try:
res = requests.get(url, timeout=timeout)
res.raise_for_status()
except (HTTPError, Timeout) as err:
# Log the error and return an empty dict if the request fails again
log.error(err)
return {}
return res.json()
def _search_rom(self, search_term: str, platform_moby_id: int) -> dict | None:
url = yarl.URL(self.games_url).with_query(
platform=[platform_moby_id or 0], title=search_term
)
roms = self._request(str(url)).get("games", [])
exact_matches = [
rom for rom in roms if rom["title"].lower() == search_term.lower()
]
return pydash.get(exact_matches or roms, "[0]", None)
def get_platform(self, slug: str) -> MobyGamesPlatform:
platform = SLUG_TO_MOBY_ID.get(slug, None)
if not platform:
return MobyGamesPlatform(moby_id=None, slug=slug)
return MobyGamesPlatform(
moby_id=platform["id"],
slug=slug,
name=platform["name"],
)
async def get_rom(self, file_name: str, platform_moby_id: int) -> MobyGamesRom:
from handler import fs_rom_handler
search_term = fs_rom_handler.get_file_name_with_no_tags(file_name)
# Support for PS2 OPL filename format
match = re.match(PS2_OPL_REGEX, file_name)
if platform_moby_id == PS2_MOBY_ID and match:
search_term = await self._ps2_opl_format(match, search_term)
# Support for switch titleID filename format
match = re.search(SWITCH_TITLEDB_REGEX, file_name)
if platform_moby_id == SWITCH_MOBY_ID and match:
search_term = await self._switch_titledb_format(match, search_term)
# Support for switch productID filename format
match = re.search(SWITCH_PRODUCT_ID_REGEX, file_name)
if platform_moby_id == SWITCH_MOBY_ID and match:
search_term = await self._switch_productid_format(match, search_term)
# Support for MAME arcade filename format
if platform_moby_id in ARCADE_MOBY_IDS:
search_term = await self._mame_format(search_term)
search_term = self.normalize_search_term(search_term)
res = self._search_rom(uc(search_term), platform_moby_id)
if not res:
return MobyGamesRom(moby_id=None)
rom = {
"moby_id": res["game_id"],
"name": res["title"],
"slug": res["moby_url"].split("/")[-1],
"summary": res.get("description", ""),
"url_cover": res.get("sample_cover.image", ""),
"url_screenshots": [s["image"] for s in res.get("sample_screenshots", [])],
}
return MobyGamesRom({k: v for k, v in rom.items() if v})
def _get_rom_by_id(self, moby_id: int) -> MobyGamesRom:
url = yarl.URL(self.games_url).with_query(id=moby_id)
roms = self._request(str(url)).get("games", [])
res = pydash.get(roms, "[0]", None)
if not res:
return MobyGamesRom(moby_id=moby_id)
rom = {
"moby_id": res["game_id"],
"name": res["title"],
"slug": res["moby_url"].split("/")[-1],
"summary": res.get("description", None),
"url_cover": res.get("sample_cover.image", None),
"url_screenshots": [s["image"] for s in res.get("sample_screenshots", [])],
}
return MobyGamesRom({k: v for k, v in rom.items() if v})
def get_matched_roms_by_id(self, moby_id: int) -> list[MobyGamesRom]:
return [self.get_rom_by_id(moby_id)]
def get_matched_roms_by_name(
self, search_term: str, platform_moby_id: int
) -> list[MobyGamesRom]:
if not platform_moby_id:
return []
url = yarl.URL(self.games_url).with_query(
platform=[platform_moby_id or 0], title=search_term
)
matched_roms = self._request(str(url))["games"]
return [
MobyGamesRom(
{
k: v
for k, v in {
"moby_id": rom["game_id"],
"name": rom["title"],
"slug": rom["moby_url"].split("/")[-1],
"summary": rom.get("description", ""),
"url_cover": rom.get("sample_cover.image", ""),
"url_screenshots": [
s["image"] for s in rom.get("sample_screenshots", [])
],
}.items()
if v
}
)
for rom in matched_roms
]
SLUG_TO_MOBY_ID: Final = {
"1292-advanced-programmable-video-system": {
"id": 253,
"name": "1292 Advanced Programmable Video System",
},
"3do": {"id": 35, "name": "3DO"},
"abc-80": {"id": 318, "name": "ABC 80"},
"apf": {"id": 213, "name": "APF MP1000/Imagination Machine"},
"acorn-32-bit": {"id": 117, "name": "Acorn 32-bit"},
"acorn-archimedes": {"id": 117, "name": "Acorn Archimedes"}, # IGDB
"adventure-vision": {"id": 210, "name": "Adventure Vision"},
"airconsole": {"id": 305, "name": "AirConsole"},
"alice-3290": {"id": 194, "name": "Alice 32/90"},
"altair-680": {"id": 265, "name": "Altair 680"},
"altair-8800": {"id": 222, "name": "Altair 8800"},
"amazon-alexa": {"id": 237, "name": "Amazon Alexa"},
"amiga": {"id": 19, "name": "Amiga"},
"amiga-cd32": {"id": 56, "name": "Amiga CD32"},
"cpc": {"id": 60, "name": "Amstrad CPC"},
"acpc": {"id": 60, "name": "Amstrad CPC"}, # IGDB
"amstrad-pcw": {"id": 136, "name": "Amstrad PCW"},
"android": {"id": 91, "name": "Android"},
"antstream": {"id": 286, "name": "Antstream"},
"apple-i": {"id": 245, "name": "Apple I"},
"apple2": {"id": 31, "name": "Apple II"},
"appleii": {"id": 31, "name": "Apple II"}, # IGDB
"apple2gs": {"id": 51, "name": "Apple IIGD"},
"apple-iigs": {"id": 51, "name": "Apple IIGD"}, # IGDB
"arcade": {"id": 143, "name": "Arcade"},
"arcadia-2001": {"id": 162, "name": "Arcadia 2001"},
"arduboy": {"id": 215, "name": "Arduboy"},
"astral-2000": {"id": 241, "name": "Astral 2000"},
"atari-2600": {"id": 28, "name": "Atari 2600"},
"atari2600": {"id": 28, "name": "Atari 2600"}, # IGDB
"atari-5200": {"id": 33, "name": "Atari 5200"},
"atari5200": {"id": 33, "name": "Atari 5200"}, # IGDB
"atari-7800": {"id": 34, "name": "Atari 7800"},
"atari7800": {"id": 34, "name": "Atari 7800"}, # IGDB
"atari-8-bit": {"id": 39, "name": "Atari 8-bit"},
"atari8bit": {"id": 39, "name": "Atari 8-bit"}, # IGDB
"atari-st": {"id": 24, "name": "Atari ST"},
"atari-vcs": {"id": 319, "name": "Atari VCS"},
"atom": {"id": 129, "name": "Atom"},
"bbc-micro": {"id": 92, "name": "BBC Micro"},
"bbcmicro": {"id": 92, "name": "BBC Micro"}, # IGDB
"brew": {"id": 63, "name": "BREW"},
"bally-astrocade": {"id": 160, "name": "Bally Astrocade"},
"astrocade": {"id": 160, "name": "Bally Astrocade"}, # IGDB
"beos": {"id": 165, "name": "BeOS"},
"blackberry": {"id": 90, "name": "BlackBerry"},
"blacknut": {"id": 290, "name": "Blacknut"},
"blu-ray-disc-player": {"id": 168, "name": "Blu-ray Player"},
"blu-ray-player": {"id": 169, "name": "Blu-ray Player"}, # IGDB
"browser": {"id": 84, "name": "Browser"},
"bubble": {"id": 231, "name": "Bubble"},
"cd-i": {"id": 73, "name": "CD-i"},
"philips-cd-i": {"id": 73, "name": "CD-i"}, # IGDB
"cdtv": {"id": 83, "name": "CDTV"},
"commodore-cdtv": {"id": 83, "name": "CDTV"}, # IGDB
"fred-cosmac": {"id": 216, "name": "COSMAC"},
"camputers-lynx": {"id": 154, "name": "Camputers Lynx"},
"cpm": {"id": 261, "name": "CP/M"},
"casio-loopy": {"id": 124, "name": "Casio Loopy"},
"casio-pv-1000": {"id": 125, "name": "Casio PV-1000"},
"casio-programmable-calculator": {
"id": 306,
"name": "Casio Programmable Calculator",
},
"champion-2711": {"id": 298, "name": "Champion 2711"},
"channel-f": {"id": 76, "name": "Channel F"},
"fairchild-channel-f": {"id": 76, "name": "Channel F"}, # IGDB
"clickstart": {"id": 188, "name": "ClickStart"},
"colecoadam": {"id": 156, "name": "Coleco Adam"},
"colecovision": {"id": 29, "name": "ColecoVision"},
"colour-genie": {"id": 197, "name": "Colour Genie"},
"c128": {"id": 61, "name": "Commodore 128"},
"commodore-16-plus4": {"id": 115, "name": "Commodore 16, Plus/4"},
"c-plus-4": {"id": 115, "name": "Commodore Plus/4"}, # IGDB
"c16": {"id": 115, "name": "Commodore 16"}, # IGDB
"c64": {"id": 27, "name": "Commodore 64"},
"pet": {"id": 77, "name": "Commodore PET/CBM"},
"cpet": {"id": 77, "name": "Commodore PET/CBM"}, # IGDB
"compal-80": {"id": 277, "name": "Compal 80"},
"compucolor-i": {"id": 243, "name": "Compucolor I"},
"compucolor-ii": {"id": 198, "name": "Compucolor II"},
"compucorp-programmable-calculator": {
"id": 238,
"name": "Compucorp Programmable Calculator",
},
"creativision": {"id": 212, "name": "CreatiVision"},
"cybervision": {"id": 301, "name": "Cybervision"},
"dos": {"id": 2, "name": "DOS"},
"dvd-player": {"id": 166, "name": "DVD Player"},
"danger-os": {"id": 285, "name": "Danger OS"},
"dedicated-console": {"id": 204, "name": "Dedicated console"},
"dedicated-handheld": {"id": 205, "name": "Dedicated handheld"},
"didj": {"id": 184, "name": "Didj"},
"doja": {"id": 72, "name": "DoJa"},
"dragon-3264": {"id": 79, "name": "Dragon 32/64"},
"dragon-32-slash-64": {"id": 79, "name": "Dragon 32/64"}, # IGDB
"dreamcast": {"id": 8, "name": "Dreamcast"},
"dc": {"id": 8, "name": "Dreamcast"}, # IGDB
"ecd-micromind": {"id": 269, "name": "ECD Micromind"},
"electron": {"id": 93, "name": "Electron"},
"acorn-electron": {"id": 93, "name": "Electron"}, # IGDB
"enterprise": {"id": 161, "name": "Enterprise"},
"epoch-cassette-vision": {"id": 137, "name": "Epoch Cassette Vision"},
"epoch-game-pocket-computer": {"id": 139, "name": "Epoch Game Pocket Computer"},
"epoch-super-cassette-vision": {"id": 138, "name": "Epoch Super Cassette Vision"},
"evercade": {"id": 284, "name": "Evercade"},
"exen": {"id": 70, "name": "ExEn"},
"exelvision": {"id": 195, "name": "Exelvision"},
"exidy-sorcerer": {"id": 176, "name": "Exidy Sorcerer"},
"fmtowns": {"id": 102, "name": "FM Towns"},
"fm-towns": {"id": 102, "name": "FM Towns"}, # IGDB
"fm-7": {"id": 126, "name": "FM-7"},
"mobile-custom": {"id": 315, "name": "Feature phone"},
"fire-os": {"id": 159, "name": "Fire OS"},
"amazon-fire-tv": {"id": 159, "name": "Fire TV"},
"freebox": {"id": 268, "name": "Freebox"},
"g-cluster": {"id": 302, "name": "G-cluster"},
"gimini": {"id": 251, "name": "GIMINI"},
"gnex": {"id": 258, "name": "GNEX"},
"gp2x": {"id": 122, "name": "GP2X"},
"gp2x-wiz": {"id": 123, "name": "GP2X Wiz"},
"gp32": {"id": 108, "name": "GP32"},
"gvm": {"id": 257, "name": "GVM"},
"galaksija": {"id": 236, "name": "Galaksija"},
"gameboy": {"id": 10, "name": "Game Boy"},
"gb": {"id": 10, "name": "Game Boy"}, # IGDB
"gameboy-advance": {"id": 12, "name": "Game Boy Advance"},
"gba": {"id": 12, "name": "Game Boy Advance"}, # IGDB
"gameboy-color": {"id": 11, "name": "Game Boy Color"},
"gbc": {"id": 11, "name": "Game Boy Color"}, # IGDB
"game-gear": {"id": 25, "name": "Game Gear"},
"gamegear": {"id": 25, "name": "Game Gear"}, # IGDB
"game-wave": {"id": 104, "name": "Game Wave"},
"game-com": {"id": 50, "name": "Game.Com"},
"game-dot-com": {"id": 50, "name": "Game.Com"}, # IGDB
"gamecube": {"id": 14, "name": "GameCube"},
"ngc": {"id": 14, "name": "GameCube"}, # IGDB
"gamestick": {"id": 155, "name": "GameStick"},
"genesis": {"id": 16, "name": "Genesis/Mega Drive"},
"genesis-slash-megadrive": {"id": 16, "name": "Genesis/Mega Drive"},
"gizmondo": {"id": 55, "name": "Gizmondo"},
"gloud": {"id": 292, "name": "Gloud"},
"glulx": {"id": 172, "name": "Glulx"},
"hd-dvd-player": {"id": 167, "name": "HD DVD Player"},
"hp-9800": {"id": 219, "name": "HP 9800"},
"hp-programmable-calculator": {"id": 234, "name": "HP Programmable Calculator"},
"heathzenith": {"id": 262, "name": "Heath/Zenith H8/H89"},
"heathkit-h11": {"id": 248, "name": "Heathkit H11"},
"hitachi-s1": {"id": 274, "name": "Hitachi S1"},
"hugo": {"id": 170, "name": "Hugo"},
"hyperscan": {"id": 192, "name": "HyperScan"},
"ibm-5100": {"id": 250, "name": "IBM 5100"},
"ideal-computer": {"id": 252, "name": "Ideal-Computer"},
"intel-8008": {"id": 224, "name": "Intel 8008"},
"intel-8080": {"id": 225, "name": "Intel 8080"},
"intel-8086": {"id": 317, "name": "Intel 8086 / 8088"},
"intellivision": {"id": 30, "name": "Intellivision"},
"interact-model-one": {"id": 295, "name": "Interact Model One"},
"interton-video-2000": {"id": 221, "name": "Interton Video 2000"},
"j2me": {"id": 64, "name": "J2ME"},
"jaguar": {"id": 17, "name": "Jaguar"},
"jolt": {"id": 247, "name": "Jolt"},
"jupiter-ace": {"id": 153, "name": "Jupiter Ace"},
"kim-1": {"id": 226, "name": "KIM-1"},
"kaios": {"id": 313, "name": "KaiOS"},
"kindle": {"id": 145, "name": "Kindle Classic"},
"laser200": {"id": 264, "name": "Laser 200"},
"laseractive": {"id": 163, "name": "LaserActive"},
"leapfrog-explorer": {"id": 185, "name": "LeapFrog Explorer"},
"leapster-explorer-slash-leadpad-explorer": {
"id": 186,
"name": "Leapster Explorer/LeapPad Explorer",
},
"leaptv": {"id": 186, "name": "LeapTV"},
"leapster": {"id": 183, "name": "Leapster"},
"linux": {"id": 1, "name": "Linux"},
"luna": {"id": 297, "name": "Luna"},
"lynx": {"id": 18, "name": "Lynx"},
"mos-technology-6502": {"id": 240, "name": "MOS Technology 6502"},
"mre": {"id": 229, "name": "MRE"},
"msx": {"id": 57, "name": "MSX"},
"macintosh": {"id": 74, "name": "Macintosh"},
"mac": {"id": 74, "name": "Macintosh"}, # IGDB
"maemo": {"id": 157, "name": "Maemo"},
"mainframe": {"id": 208, "name": "Mainframe"},
"matsushitapanasonic-jr": {"id": 307, "name": "Matsushita/Panasonic JR"},
"mattel-aquarius": {"id": 135, "name": "Mattel Aquarius"},
"meego": {"id": 158, "name": "MeeGo"},
"memotech-mtx": {"id": 148, "name": "Memotech MTX"},
"meritum": {"id": 311, "name": "Meritum"},
"microbee": {"id": 200, "name": "Microbee"},
"microtan-65": {"id": 232, "name": "Microtan 65"},
"microvision": {"id": 97, "name": "Microvision"},
"microvision--1": {"id": 97, "name": "Microvision"}, # IGDB
"mophun": {"id": 71, "name": "Mophun"},
"motorola-6800": {"id": 235, "name": "Motorola 6800"},
"motorola-68k": {"id": 275, "name": "Motorola 68k"},
"ngage": {"id": 32, "name": "N-Gage"},
"ngage2": {"id": 89, "name": "N-Gage (service)"},
"nes": {"id": 22, "name": "NES"},
"nascom": {"id": 175, "name": "Nascom"},
"neo-geo": {"id": 36, "name": "Neo Geo"},
"neogeoaes": {"id": 36, "name": "Neo Geo"}, # IGDB
"neogeomvs": {"id": 36, "name": "Neo Geo"}, # IGDB
"neo-geo-cd": {"id": 54, "name": "Neo Geo CD"},
"neo-geo-pocket": {"id": 52, "name": "Neo Geo Pocket"},
"neo-geo-pocket-color": {"id": 53, "name": "Neo Geo Pocket Color"},
"neo-geo-x": {"id": 279, "name": "Neo Geo X"},
"new-nintendo-3ds": {"id": 174, "name": "New Nintendo 3DS"},
"newbrain": {"id": 177, "name": "NewBrain"},
"newton": {"id": 207, "name": "Newton"},
"3ds": {"id": 101, "name": "Nintendo 3DS"},
"n64": {"id": 9, "name": "Nintendo 64"},
"nintendo-ds": {"id": 44, "name": "Nintendo DS"},
"nds": {"id": 44, "name": "Nintendo DS"}, # IGDB
"nintendo-dsi": {"id": 87, "name": "Nintendo DSi"},
"switch": {"id": 203, "name": "Nintendo Switch"},
"northstar": {"id": 266, "name": "North Star"},
"noval-760": {"id": 244, "name": "Noval 760"},
"nuon": {"id": 116, "name": "Nuon"},
"ooparts": {"id": 300, "name": "OOParts"},
"os2": {"id": 146, "name": "OS/2"},
"oculus-go": {"id": 218, "name": "Oculus Go"},
"odyssey": {"id": 75, "name": "Odyssey"},
"odyssey--1": {"id": 75, "name": "Odyssey"}, # IGDB
"odyssey-2": {"id": 78, "name": "Odyssey 2"},
"odyssey-2-slash-videopac-g7000": {"id": 78, "name": "Odyssey 2/Videopac G7000"},
"ohio-scientific": {"id": 178, "name": "Ohio Scientific"},
"onlive": {"id": 282, "name": "OnLive"},
"onlive-game-system": {"id": 282, "name": "OnLive Game System"}, # IGDB
"orao": {"id": 270, "name": "Orao"},
"oric": {"id": 111, "name": "Oric"},
"ouya": {"id": 144, "name": "Ouya"},
"pc-booter": {"id": 4, "name": "PC Booter"},
"pc-6001": {"id": 149, "name": "PC-6001"},
"pc-8000": {"id": 201, "name": "PC-8000"},
"pc88": {"id": 94, "name": "PC-88"},
"pc-8800-series": {"id": 94, "name": "PC-8800 Series"}, # IGDB
"pc98": {"id": 95, "name": "PC-98"},
"pc-9800-series": {"id": 95, "name": "PC-9800 Series"}, # IGDB
"pc-fx": {"id": 59, "name": "PC-FX"},
"pico": {"id": 316, "name": "PICO"},
"ps-vita": {"id": 105, "name": "PS Vita"},
"psvita": {"id": 105, "name": "PS Vita"}, # IGDB
"psp": {"id": 46, "name": "PSP"},
"palmos": {"id": 65, "name": "Palm OS"},
"palm-os": {"id": 65, "name": "Palm OS"}, # IGDB
"pandora": {"id": 308, "name": "Pandora"},
"pebble": {"id": 304, "name": "Pebble"},
"philips-vg-5000": {"id": 133, "name": "Philips VG 5000"},
"photocd": {"id": 272, "name": "Photo CD"},
"pippin": {"id": 112, "name": "Pippin"},
"playstation": {"id": 6, "name": "PlayStation"},
"ps": {"id": 6, "name": "PlayStation"}, # IGDB
"ps2": {"id": 7, "name": "PlayStation 2"},
"ps3": {"id": 81, "name": "PlayStation 3"},
"playstation-4": {"id": 141, "name": "PlayStation 4"},
"ps4--1": {"id": 141, "name": "PlayStation 4"}, # IGDB
"playstation-5": {"id": 288, "name": "PlayStation 5"},
"ps5": {"id": 288, "name": "PlayStation 5"}, # IGDB
"playstation-now": {"id": 294, "name": "PlayStation Now"},
"playdate": {"id": 303, "name": "Playdate"},
"playdia": {"id": 107, "name": "Playdia"},
"plex-arcade": {"id": 291, "name": "Plex Arcade"},
"pokitto": {"id": 230, "name": "Pokitto"},
"pokemon-mini": {"id": 152, "name": "Pokémon Mini"},
"poly-88": {"id": 249, "name": "Poly-88"},
"oculus-quest": {"id": 271, "name": "Quest"},
"rca-studio-ii": {"id": 113, "name": "RCA Studio II"},
"research-machines-380z": {"id": 309, "name": "Research Machines 380Z"},
"roku": {"id": 196, "name": "Roku"},
"sam-coupe": {"id": 120, "name": "SAM Coupé"},
"scmp": {"id": 255, "name": "SC/MP"},
"sd-200270290": {"id": 267, "name": "SD-200/270/290"},
"sega-32x": {"id": 21, "name": "SEGA 32X"},
"sega32": {"id": 21, "name": "SEGA 32X"}, # IGDB
"sega-cd": {"id": 20, "name": "SEGA CD"},
"segacd": {"id": 20, "name": "SEGA CD"}, # IGDB
"sega-master-system": {"id": 26, "name": "SEGA Master System"},
"sega-pico": {"id": 103, "name": "SEGA Pico"},
"sega-saturn": {"id": 23, "name": "SEGA Saturn"},
"saturn": {"id": 23, "name": "SEGA Saturn"}, # IGDB
"sg-1000": {"id": 114, "name": "SG-1000"},
"sk-vm": {"id": 259, "name": "SK-VM"},
"smc-777": {"id": 273, "name": "SMC-777"},
"snes": {"id": 15, "name": "SNES"},
"sri-5001000": {"id": 242, "name": "SRI-500/1000"},
"swtpc-6800": {"id": 228, "name": "SWTPC 6800"},
"sharp-mz-80b20002500": {"id": 182, "name": "Sharp MZ-80B/2000/2500"},
"sharp-mz-2200": {"id": 180, "name": "Sharp MZ-2200"},
"sharp-mz-80k7008001500": {"id": 181, "name": "Sharp MZ-80K/700/800/1500"},
"sharp-x1": {"id": 121, "name": "Sharp X1"},
"x1": {"id": 121, "name": "Sharp X1"}, # IGDB
"sharp-x68000": {"id": 106, "name": "Sharp X68000"},
"sharp-zaurus": {"id": 202, "name": "Sharp Zaurus"},
"signetics-2650": {"id": 278, "name": "Signetics 2650"},
"sinclair-ql": {"id": 131, "name": "Sinclair QL"},
"socrates": {"id": 190, "name": "Socrates"},
"sol-20": {"id": 199, "name": "Sol-20"},
"sord-m5": {"id": 134, "name": "Sord M5"},
"spectravideo": {"id": 85, "name": "Spectravideo"},
"stadia": {"id": 281, "name": "Stadia"},
"super-acan": {"id": 110, "name": "Super A'can"},
"super-vision-8000": {"id": 296, "name": "Super Vision 8000"},
"supergrafx": {"id": 127, "name": "SuperGrafx"},
"supervision": {"id": 109, "name": "Supervision"},
"sure-shot-hd": {"id": 287, "name": "Sure Shot HD"},
"symbian": {"id": 67, "name": "Symbian"},
"tads": {"id": 171, "name": "TADS"},
"ti-programmable-calculator": {"id": 239, "name": "TI Programmable Calculator"},
"ti-994a": {"id": 47, "name": "TI-99/4A"},
"ti-99": {"id": 47, "name": "TI-99/4A"},
"tim": {"id": 246, "name": "TIM"},
"trs-80": {"id": 58, "name": "TRS-80"},
"trs-80-coco": {"id": 62, "name": "TRS-80 Color Computer"},
"trs-80-color-computer": {"id": 62, "name": "TRS-80 Color Computer"}, # IGDB
"trs-80-mc-10": {"id": 193, "name": "TRS-80 MC-10"},
"trs-80-model-100": {"id": 312, "name": "TRS-80 Model 100"},
"taito-x-55": {"id": 283, "name": "Taito X-55"},
"tatung-einstein": {"id": 150, "name": "Tatung Einstein"},
"tektronix-4050": {"id": 223, "name": "Tektronix 4050"},
"tele-spiel": {"id": 220, "name": "Tele-Spiel ES-2201"},
"telstar-arcade": {"id": 233, "name": "Telstar Arcade"},
"terminal": {"id": 209, "name": "Terminal"},
"thomson-mo": {"id": 147, "name": "Thomson MO"},
"thomson-mo5": {"id": 147, "name": "Thomson MO5"},
"thomson-to": {"id": 130, "name": "Thomson TO"},
"tiki-100": {"id": 263, "name": "Tiki 100"},
"timex-sinclair-2068": {"id": 173, "name": "Timex Sinclair 2068"},
"tizen": {"id": 206, "name": "Tizen"},
"tomahawk-f1": {"id": 256, "name": "Tomahawk F1"},
"tomy-tutor": {"id": 151, "name": "Tomy Tutor"},
"triton": {"id": 310, "name": "Triton"},
"turbografx-cd": {"id": 45, "name": "TurboGrafx CD"},
"turbografx-16-slash-pc-engine-cd": {"id": 45, "name": "TurboGrafx CD"},
"turbo-grafx": {"id": 40, "name": "TurboGrafx-16"},
"turbografx16--1": {"id": 40, "name": "TurboGrafx-16"}, # IGDB
"vflash": {"id": 189, "name": "V.Flash"},
"vsmile": {"id": 42, "name": "V.Smile"},
"vic-20": {"id": 43, "name": "VIC-20"},
"vis": {"id": 164, "name": "VIS"},
"vectrex": {"id": 37, "name": "Vectrex"},
"versatile": {"id": 299, "name": "Versatile"},
"videobrain": {"id": 214, "name": "VideoBrain"},
"videopac-g7400": {"id": 128, "name": "Videopac+ G7400"},
"virtual-boy": {"id": 38, "name": "Virtual Boy"},
"virtualboy": {"id": 38, "name": "Virtual Boy"},
"wipi": {"id": 260, "name": "WIPI"},
"wang2200": {"id": 217, "name": "Wang 2200"},
"wii": {"id": 82, "name": "Wii"},
"wii-u": {"id": 132, "name": "Wii U"},
"wiiu": {"id": 132, "name": "Wii U"},
"windows": {"id": 3, "name": "Windows"},
"win": {"id": 3, "name": "Windows"}, # IGDB
"win3x": {"id": 5, "name": "Windows 3.x"},
"windows-apps": {"id": 140, "name": "Windows Apps"},
"windowsmobile": {"id": 66, "name": "Windows Mobile"},
"windows-mobile": {"id": 66, "name": "Windows Mobile"}, # IGDB
"windows-phone": {"id": 98, "name": "Windows Phone"},
"winphone": {"id": 98, "name": "Windows Phone"}, # IGDB
"wonderswan": {"id": 48, "name": "WonderSwan"},
"wonderswan-color": {"id": 49, "name": "WonderSwan Color"},
"xavixport": {"id": 191, "name": "XaviXPORT"},
"xbox": {"id": 13, "name": "Xbox"},
"xbox360": {"id": 69, "name": "Xbox 360"},
"xboxcloudgaming": {"id": 293, "name": "Xbox Cloud Gaming"},
"xbox-one": {"id": 142, "name": "Xbox One"},
"xboxone": {"id": 142, "name": "Xbox One"},
"xbox-series": {"id": 289, "name": "Xbox Series"},
"series-x": {"id": 289, "name": "Xbox Series X"}, # IGDB
"xerox-alto": {"id": 254, "name": "Xerox Alto"},
"z-machine": {"id": 169, "name": "Z-machine"},
"zx-spectrum": {"id": 41, "name": "ZX Spectrum"},
"zx-spectrum-next": {"id": 280, "name": "ZX Spectrum Next"},
"zx80": {"id": 118, "name": "ZX80"},
"zx81": {"id": 119, "name": "ZX81"},
"sinclair-zx81": {"id": 119, "name": "ZX81"}, # IGDB
"zeebo": {"id": 88, "name": "Zeebo"},
"z80": {"id": 227, "name": "Zilog Z80"},
"zilog-z8000": {"id": 276, "name": "Zilog Z8000"},
"zodiac": {"id": 68, "name": "Zodiac"},
"zune": {"id": 211, "name": "Zune"},
"bada": {"id": 99, "name": "bada"},
"digiblast": {"id": 187, "name": "digiBlast"},
"ipad": {"id": 96, "name": "iPad"},
"iphone": {"id": 86, "name": "iPhone"},
"ios": {"id": 86, "name": "iOS"},
"ipod-classic": {"id": 80, "name": "iPod Classic"},
"iircade": {"id": 314, "name": "iiRcade"},
"tvos": {"id": 179, "name": "tvOS"},
"watchos": {"id": 180, "name": "watchOS"},
"webos": {"id": 100, "name": "webOS"},
}
# Reverse lookup
MOBY_ID_TO_SLUG = {v["id"]: k for k, v in SLUG_TO_MOBY_ID.items()}

View File

@@ -8,6 +8,7 @@ from handler import (
fs_resource_handler,
fs_rom_handler,
igdb_handler,
moby_handler,
)
from logger.logger import log
from models.assets import Save, Screenshot, State
@@ -35,7 +36,7 @@ def _get_main_platform_igdb_id(platform: Platform):
return main_platform_igdb_id
def scan_platform(fs_slug: str, fs_platforms) -> Platform:
def scan_platform(fs_slug: str, fs_platforms: list[str]) -> Platform:
"""Get platform details
Args:
@@ -67,16 +68,18 @@ def scan_platform(fs_slug: str, fs_platforms) -> Platform:
except (KeyError, TypeError, AttributeError):
platform_attrs["slug"] = fs_slug
platform = igdb_handler.get_platform(platform_attrs["slug"])
igdb_platform = igdb_handler.get_platform(platform_attrs["slug"])
moby_platform = moby_handler.get_platform(platform_attrs["slug"])
if platform["igdb_id"]:
log.info(emoji.emojize(f" Identified as {platform['name']} :video_game:"))
else:
log.warning(
emoji.emojize(f" {platform_attrs['slug']} not found in IGDB :cross_mark:")
platform_attrs["name"] = platform_attrs["slug"].replace("-", " ").title()
platform_attrs.update({**moby_platform, **igdb_platform}) # Reverse order
if platform_attrs["igdb_id"] or platform_attrs["moby_id"]:
log.info(
emoji.emojize(f" Identified as {platform_attrs['name']} :video_game:")
)
platform_attrs.update(platform)
else:
log.warning(emoji.emojize(f" {platform_attrs['slug']} not found :cross_mark:"))
return Platform(**platform_attrs)
@@ -84,18 +87,16 @@ def scan_platform(fs_slug: str, fs_platforms) -> Platform:
async def scan_rom(
platform: Platform,
rom_attrs: dict,
r_igbd_id_search: str = "",
overwrite: bool = False,
) -> Rom:
roms_path = fs_rom_handler.get_fs_structure(platform.fs_slug)
log.info(f"\t · {r_igbd_id_search or rom_attrs['file_name']}")
log.info(f"\t · {rom_attrs['file_name']}")
if rom_attrs.get("multi", False):
for file in rom_attrs["files"]:
log.info(f"\t\t · {file}")
# Update properties that don't require IGDB
# Update properties that don't require metadata
file_size = fs_rom_handler.get_rom_file_size(
multi=rom_attrs["multi"],
file_name=rom_attrs["file_name"],
@@ -105,6 +106,13 @@ async def scan_rom(
regs, rev, langs, other_tags = fs_rom_handler.parse_tags(rom_attrs["file_name"])
rom_attrs.update(
{
"igdb_id": None,
"moby_id": None,
"slug": "",
"name": rom_attrs["file_name"],
"summary": "",
"url_cover": "",
"url_screenshots": [],
"platform_id": platform.id,
"file_path": roms_path,
"file_name": rom_attrs["file_name"],
@@ -127,33 +135,29 @@ async def scan_rom(
)
main_platform_igdb_id = _get_main_platform_igdb_id(platform)
# Search in IGDB
igdb_handler_rom = (
igdb_handler.get_rom_by_id(int(r_igbd_id_search))
if r_igbd_id_search
else await igdb_handler.get_rom(rom_attrs["file_name"], main_platform_igdb_id)
igdb_handler_rom = await igdb_handler.get_rom(
rom_attrs["file_name"], main_platform_igdb_id
)
moby_handler_rom = await moby_handler.get_rom(
rom_attrs["file_name"], platform.moby_id
)
rom_attrs.update(igdb_handler_rom)
# Return early if not found in IGDB
if not igdb_handler_rom["igdb_id"]:
# Return early if not found in IGDB or MobyGames
if not igdb_handler_rom["igdb_id"] and not moby_handler_rom["moby_id"]:
log.warning(
emoji.emojize(
f"\t {r_igbd_id_search or rom_attrs['file_name']} not found in IGDB :cross_mark:"
f"\t {rom_attrs['file_name']} not found :cross_mark:"
)
)
return Rom(**rom_attrs)
log.info(
emoji.emojize(f"\t Identified as {igdb_handler_rom['name']} :alien_monster:")
)
rom_attrs.update({**moby_handler_rom, **igdb_handler_rom}) # Reversed to prioritize IGDB
log.info(emoji.emojize(f"\t Identified as {rom_attrs['name']} :alien_monster:"))
# Update properties from IGDB
rom_attrs.update(
fs_resource_handler.get_rom_cover(
overwrite=overwrite,
overwrite=False,
platform_fs_slug=platform.slug,
rom_name=rom_attrs["name"],
url_cover=rom_attrs["url_cover"],

View File

@@ -10,6 +10,7 @@ class Platform(BaseModel):
id = Column(Integer(), primary_key=True, autoincrement=True)
igdb_id: int = Column(Integer())
sgdb_id: int = Column(Integer())
moby_id: int = Column(Integer())
slug: str = Column(String(length=50), nullable=False)
fs_slug: str = Column(String(length=50), nullable=False)
name: str = Column(String(length=400))

View File

@@ -24,6 +24,7 @@ class Rom(BaseModel):
igdb_id: int = Column(Integer())
sgdb_id: int = Column(Integer())
moby_id: int = Column(Integer())
file_name: str = Column(String(length=450), nullable=False)
file_name_no_tags: str = Column(String(length=450), nullable=False)
@@ -36,6 +37,7 @@ class Rom(BaseModel):
slug: str = Column(String(length=400))
summary: str = Column(Text)
igdb_metadata: MySQLJSON = Column(MySQLJSON, default=dict)
moby_metadata: MySQLJSON = Column(MySQLJSON, default=dict)
path_cover_s: str = Column(Text, default="")
path_cover_l: str = Column(Text, default="")
@@ -47,7 +49,9 @@ class Rom(BaseModel):
tags: JSON = Column(JSON, default=[])
path_screenshots: JSON = Column(JSON, default=[])
url_screenshots: JSON = Column(JSON, default=[], doc="URLs to screenshots stored in IGDB")
url_screenshots: JSON = Column(
JSON, default=[], doc="URLs to screenshots stored in IGDB"
)
multi: bool = Column(Boolean, default=False)
files: JSON = Column(JSON, default=[])
@@ -112,40 +116,52 @@ class Rom(BaseModel):
Rom.igdb_id == self.igdb_id,
)
).all()
# Metadata fields
@property
def total_rating(self) -> str:
return self.igdb_metadata.get("total_rating", "")
return (
self.igdb_metadata.get("total_rating", None)
or self.moby_metadata.get("moby_score", None)
or ""
)
@property
def aggregated_rating(self) -> str:
return self.igdb_metadata.get("aggregated_rating", "")
@property
def alternative_names(self) -> list[str]:
return self.igdb_metadata.get("alternative_names", [])
return (
self.igdb_metadata.get("alternative_names", None)
or self.moby_metadata.get("alternate_titles", None)
or []
)
@property
def first_release_date(self) -> int:
return self.igdb_metadata.get("first_release_date", 0)
@property
def genres(self) -> list[str]:
return self.igdb_metadata.get("genres", [])
return (
self.igdb_metadata.get("genres", None)
or self.moby_metadata.get("genres", None)
or []
)
@property
def franchises(self) -> list[str]:
return self.igdb_metadata.get("franchises", [])
@property
def collections(self) -> list[str]:
return self.igdb_metadata.get("collections", [])
return self.igdb_metadata.get("collections", [])
@property
def companies(self) -> list[str]:
return self.igdb_metadata.get("companies", [])
@property
def game_modes(self) -> list[str]:
return self.igdb_metadata.get("game_modes", [])

View File

@@ -6,6 +6,9 @@ ROMM_HOST=localhost
IGDB_CLIENT_ID=
IGDB_CLIENT_SECRET=
# Mobygames
MOBYGAMES_API_KEY=
# Database config
DB_HOST=127.0.0.1
DB_PORT=3306

View File

@@ -17,6 +17,7 @@ services:
- DB_PASSWD= # Should match MYSQL_PASSWORD in mariadb
- IGDB_CLIENT_ID= # Generate an ID and SECRET in IGDB
- IGDB_CLIENT_SECRET= # https://api-docs.igdb.com/#account-creation
- MOBYGAMES_API_KEY= # https://www.mobygames.com/info/api/
- ROMM_AUTH_SECRET_KEY= # Generate a key with `openssl rand -hex 32`
- ROMM_AUTH_USERNAME=admin
- ROMM_AUTH_PASSWORD= # default: admin

View File

@@ -15,8 +15,6 @@ export type { ConfigResponse } from './models/ConfigResponse';
export type { CursorPage_RomSchema_ } from './models/CursorPage_RomSchema_';
export type { HeartbeatResponse } from './models/HeartbeatResponse';
export type { HTTPValidationError } from './models/HTTPValidationError';
export type { IGDBMetadata } from './models/IGDBMetadata';
export type { IGDBPlatform } from './models/IGDBPlatform';
export type { IGDBRelatedGame } from './models/IGDBRelatedGame';
export type { MessageResponse } from './models/MessageResponse';
export type { PlatformSchema } from './models/PlatformSchema';

View File

@@ -1,28 +0,0 @@
/* generated using openapi-typescript-codegen -- do no edit */
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import type { IGDBPlatform } from './IGDBPlatform';
import type { IGDBRelatedGame } from './IGDBRelatedGame';
export type IGDBMetadata = {
total_rating: string;
aggregated_rating: string;
first_release_date: (number | null);
genres: Array<string>;
franchises: Array<string>;
alternative_names: Array<string>;
collections: Array<string>;
companies: Array<string>;
game_modes: Array<string>;
platforms: Array<IGDBPlatform>;
expansions: Array<IGDBRelatedGame>;
dlcs: Array<IGDBRelatedGame>;
remasters: Array<IGDBRelatedGame>;
remakes: Array<IGDBRelatedGame>;
expanded_games: Array<IGDBRelatedGame>;
ports: Array<IGDBRelatedGame>;
similar_games: Array<IGDBRelatedGame>;
};

View File

@@ -1,10 +0,0 @@
/* generated using openapi-typescript-codegen -- do no edit */
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type IGDBPlatform = {
igdb_id: number;
name: string;
};

View File

@@ -9,6 +9,7 @@ export type PlatformSchema = {
fs_slug: string;
igdb_id?: (number | null);
sgdb_id?: (number | null);
moby_id?: (number | null);
name: (string | null);
logo_path: string;
rom_count: number;

View File

@@ -12,6 +12,7 @@ export type RomSchema = {
id: number;
igdb_id: (number | null);
sgdb_id: (number | null);
moby_id: (number | null);
platform_id: number;
platform_slug: string;
platform_name: string;

View File

@@ -3,15 +3,13 @@
/* tslint:disable */
/* eslint-disable */
import type { IGDBMetadata } from './IGDBMetadata';
export type SearchRomSchema = {
igdb_id: (number | null);
name: (string | null);
slug: (string | null);
summary: (string | null);
igdb_id?: (number | null);
moby_id?: (number | null);
slug: string;
name: string;
summary: string;
url_cover: string;
url_screenshots: Array<string>;
igdb_metadata: (IGDBMetadata | null);
};

View File

@@ -25,7 +25,7 @@ const galleryFilter = storeGalleryFilter();
<v-divider class="mx-2 my-4" />
<v-row no-gutters>
<v-col class="text-caption">
<p>{{ rom.summary }}</p>
<p v-html="rom.summary"></p>
</v-col>
</v-row>
</template>

View File

@@ -31,7 +31,7 @@ defineProps<{ rom: Rom }>();
<v-chip variant="outlined" class="text-romm-accent-1" @click="" label>
<a
style="text-decoration: none; color: inherit"
href=https://www.igdb.com
href="https://www.igdb.com/games/"
target="_blank"
>IGDB</a
>
@@ -53,10 +53,41 @@ defineProps<{ rom: Rom }>();
</v-chip>
</v-col>
</v-row>
<v-row
v-if="rom.moby_id"
class="align-center justify-center pa-2"
no-gutters
>
<v-col class="text-center">
<v-chip variant="outlined" class="text-romm-accent-1" @click="" label>
<a
style="text-decoration: none; color: inherit"
href="https://www.mobygames.com/game/"
target="_blank"
>MobyGames</a
>
</v-chip>
</v-col>
<v-col class="text-center">
<v-chip variant="tonal" class="text-romm-accent-1" @click="" label>
<a
style="text-decoration: none; color: inherit"
:href="`http://www.mobygames.com/game/${rom.moby_id}`"
target="_blank"
>{{ rom.moby_id }}</a
>
</v-chip>
</v-col>
<v-col class="text-center">
<v-chip variant="text" label>
{{ rom.total_rating }}
</v-chip>
</v-col>
</v-row>
</div>
</template>
<style scoped>
.table {
border: 1px solid rgba(var(--v-theme-primary));
}
</style>
</style>

View File

@@ -74,6 +74,7 @@ const { smAndDown } = useDisplay();
</v-col>
</v-row>
<v-row
v-if="rom.igdb_id"
class="text-white text-shadow"
:class="{ 'text-center': smAndDown }"
no-gutters
@@ -94,6 +95,28 @@ const { smAndDown } = useDisplay();
</a>
</v-col>
</v-row>
<v-row
v-if="rom.moby_id"
class="text-white text-shadow"
:class="{ 'text-center': smAndDown }"
no-gutters
>
<v-col cols="12">
<a
style="text-decoration: none; color: inherit"
:href="`http://www.mobygames.com/game/${rom.moby_id}`"
target="_blank"
>
<v-chip size="x-small" @click="">
<span>MobyGames</span>
<v-divider class="mx-2 border-opacity-25" vertical />
<span>ID: {{ rom.moby_id }}</span>
<v-divider class="mx-2 border-opacity-25" vertical />
<span>Rating: {{ rom.total_rating }}</span>
</v-chip>
</a>
</v-col>
</v-row>
</template>
<style scoped>

View File

@@ -44,7 +44,6 @@ async function searchRom() {
await romApi
.searchRom({
romId: rom.value.id,
source: "igdb",
searchTerm: searchTerm.value,
searchBy: searchBy.value,
searchExtended: searchExtended.value,

View File

@@ -21,10 +21,14 @@ defineProps<{ platform: Platform; rail: boolean }>();
location="bottom"
class="tooltip"
transition="fade-transition"
text="Not found in IGDB"
text="Not found"
open-delay="500"
><template v-slot:activator="{ props }">
<div v-bind="props" class="igdb-icon" v-if="!platform.igdb_id">
<div
v-bind="props"
class="not-found-icon"
v-if="!platform.igdb_id && !platform.moby_id"
>
</div></template
></v-tooltip
@@ -40,7 +44,7 @@ defineProps<{ platform: Platform; rail: boolean }>();
</template>
<style scoped>
.igdb-icon {
.not-found-icon {
position: absolute;
bottom: 0;
right: 0;

View File

@@ -81,13 +81,11 @@ function clearRomFromDownloads({ id }: { id: number }) {
async function searchRom({
romId,
source,
searchTerm,
searchBy,
searchExtended: searchExtended,
}: {
romId: number;
source: string;
searchTerm: string;
searchBy: string;
searchExtended: boolean;
@@ -95,7 +93,6 @@ async function searchRom({
return api.get("/search/roms", {
params: {
rom_id: romId,
source: source,
search_term: searchTerm,
search_by: searchBy,
search_extended: searchExtended,

View File

@@ -52,11 +52,11 @@ export default defineStore("roms", {
return;
}
// Group roms by igdb_id
// Group roms by external id
this._grouped = Object.values(
groupBy(this._all, (game) =>
// If igdb_id is null, generate a random id so that the roms are not grouped
isNull(game.igdb_id) ? nanoid() : game.igdb_id
// If external id is null, generate a random id so that the roms are not grouped
game.igdb_id || game.moby_id || nanoid()
)
)
.map((games) => ({

View File

@@ -117,11 +117,11 @@ async function scan() {
class="text-body-2 romm-grey"
:to="{ name: 'rom', params: { rom: rom.id } }"
>
<span v-if="rom.igdb_id" class="ml-10">
<span v-if="rom.igdb_id || rom.moby_id" class="ml-10">
Identified <b>{{ rom.name }} 👾</b>
</span>
<span v-else class="ml-10">
{{ rom.file_name }} not found in IGDB
{{ rom.file_name }} not found
</span>
</v-list-item>
</v-col>