diff --git a/backend/alembic/versions/0050_firmware_add_is_verified.py b/backend/alembic/versions/0050_firmware_add_is_verified.py new file mode 100644 index 000000000..29f0885d5 --- /dev/null +++ b/backend/alembic/versions/0050_firmware_add_is_verified.py @@ -0,0 +1,55 @@ +"""Add is_verified column to firmware table + +Revision ID: 0050_firmware_add_is_verified +Revises: 0049_add_fs_size_bytes +Create Date: 2025-08-22 04:42:22.367888 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0050_firmware_add_is_verified" +down_revision = "0049_add_fs_size_bytes" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with op.batch_alter_table("firmware", schema=None) as batch_op: + batch_op.add_column(sa.Column("is_verified", sa.Boolean(), nullable=True)) + + # Calculate is_verified for all firmware + from handler.database import db_firmware_handler + from models.firmware import Firmware + + all_firmware = db_firmware_handler.list_firmware() + verified_firmware_ids = [] + + for firmware in all_firmware: + is_verified = Firmware.verify_file_hashes( + platform_slug=firmware.platform.slug, + file_name=firmware.file_name, + file_size_bytes=firmware.file_size_bytes, + md5_hash=firmware.md5_hash, + sha1_hash=firmware.sha1_hash, + crc_hash=firmware.crc_hash, + ) + if is_verified: + verified_firmware_ids.append(firmware.id) + + op.execute("UPDATE firmware SET is_verified = 0") + if verified_firmware_ids: + op.execute( + # trunk-ignore(bandit/B608) + f"UPDATE firmware SET is_verified = 1 WHERE id IN ({','.join(map(str, verified_firmware_ids))})" + ) + + with op.batch_alter_table("firmware", schema=None) as batch_op: + batch_op.alter_column("is_verified", existing_type=sa.Boolean(), nullable=False) + + +def downgrade() -> None: + with op.batch_alter_table("firmware", schema=None) as batch_op: + batch_op.drop_column("is_verified") diff --git a/backend/endpoints/firmware.py b/backend/endpoints/firmware.py index bb4014b45..906872a17 100644 --- a/backend/endpoints/firmware.py +++ b/backend/endpoints/firmware.py @@ -11,6 +11,7 @@ from handler.scan_handler import scan_firmware from logger.formatter import BLUE from logger.formatter import highlight as hl from logger.logger import log +from models.firmware import Firmware from utils.router import APIRouter router = APIRouter( @@ -69,13 +70,27 @@ async def add_firmware( firmware=db_firmware, ) + is_verified = Firmware.verify_file_hashes( + platform_slug=db_platform.slug, + file_name=file.filename, + file_size_bytes=scanned_firmware.file_size_bytes, + md5_hash=scanned_firmware.md5_hash, + sha1_hash=scanned_firmware.sha1_hash, + crc_hash=scanned_firmware.crc_hash, + ) + if db_firmware: db_firmware_handler.update_firmware( - db_firmware.id, {"file_size_bytes": scanned_firmware.file_size_bytes} + db_firmware.id, + { + "file_size_bytes": scanned_firmware.file_size_bytes, + "is_verified": is_verified, + }, ) continue scanned_firmware.platform_id = db_platform.id + scanned_firmware.is_verified = is_verified db_firmware_handler.add_firmware(scanned_firmware) uploaded_firmware.append(scanned_firmware) @@ -239,7 +254,7 @@ async def delete_firmware( file_path = f"{firmware.file_path}/{firmware.file_name}" await fs_firmware_handler.remove_file(file_path=file_path) except FileNotFoundError: - error = f"Firmware file {hl(firmware.file_name)} not found for platform {hl(firmware.platform_slug)}" + error = f"Firmware file {hl(firmware.file_name)} not found for platform {hl(firmware.platform.slug)}" log.error(error) errors.append(error) failed_items += 1 diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index 566d7350e..d3fe5c7a9 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -33,6 +33,7 @@ from handler.socket_handler import socket_handler from logger.formatter import BLUE, LIGHTYELLOW from logger.formatter import highlight as hl from logger.logger import log +from models.firmware import Firmware from models.platform import Platform from models.rom import Rom, RomFile from rq import Worker @@ -95,11 +96,22 @@ async def _identify_firmware( firmware=firmware, ) + is_verified = Firmware.verify_file_hashes( + platform_slug=platform.slug, + file_name=fs_fw, + file_size_bytes=scanned_firmware.file_size_bytes, + md5_hash=scanned_firmware.md5_hash, + sha1_hash=scanned_firmware.sha1_hash, + crc_hash=scanned_firmware.crc_hash, + ) + scan_stats.scanned_firmware += 1 scan_stats.added_firmware += 1 if not firmware else 0 scanned_firmware.missing_from_fs = False + scanned_firmware.is_verified = is_verified db_firmware_handler.add_firmware(scanned_firmware) + return scan_stats diff --git a/backend/handler/database/base_handler.py b/backend/handler/database/base_handler.py index 5c921f5bd..a78b3d84d 100644 --- a/backend/handler/database/base_handler.py +++ b/backend/handler/database/base_handler.py @@ -1,3 +1,4 @@ +import logging import time from config import DEV_SQL_ECHO @@ -6,10 +7,13 @@ from sqlalchemy import create_engine, event from sqlalchemy.orm import sessionmaker sync_engine = create_engine( - ConfigManager.get_db_engine(), pool_pre_ping=True, echo=DEV_SQL_ECHO + ConfigManager.get_db_engine(), pool_pre_ping=True, echo=False ) sync_session = sessionmaker(bind=sync_engine, expire_on_commit=False) +# Disable SQLAlchemy logging as echo will print the queries +logging.getLogger("sqlalchemy.engine.Engine").handlers = [logging.NullHandler()] + if DEV_SQL_ECHO: @@ -18,6 +22,7 @@ if DEV_SQL_ECHO: conn, cursor, statement, parameters, context, executemany ): context._query_start_time = time.time() + print("--------START--------") print(f"SQL: {statement}") print(f"Parameters: {parameters}") @@ -25,6 +30,7 @@ if DEV_SQL_ECHO: def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): total_time = time.time() - context._query_start_time print(f"Execution time: {total_time:.4f} seconds") + print("--------END--------") class DBBaseHandler: ... diff --git a/backend/handler/database/collections_handler.py b/backend/handler/database/collections_handler.py index cbf20da29..4b9abed3c 100644 --- a/backend/handler/database/collections_handler.py +++ b/backend/handler/database/collections_handler.py @@ -11,7 +11,7 @@ from models.collection import ( ) from models.rom import Rom from sqlalchemy import delete, insert, literal, or_, select, update -from sqlalchemy.orm import Query, Session, selectinload +from sqlalchemy.orm import Query, Session, noload, selectinload from .base_handler import DBBaseHandler @@ -20,7 +20,13 @@ def with_roms(func): @functools.wraps(func) def wrapper(*args, **kwargs): kwargs["query"] = select(Collection).options( - selectinload(Collection.roms), + selectinload(Collection.roms) + .load_only( + Rom.id, + Rom.path_cover_s, + Rom.path_cover_l, + ) + .options(noload(Rom.platform), noload(Rom.metadatum)) ) return func(*args, **kwargs) diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index 39c56e603..269d1f2fb 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -5,6 +5,7 @@ from typing import Any from config import ROMM_DB_DRIVER from decorators.database import begin_session from handler.metadata.base_hander import UniversalPlatformSlug as UPS +from models.assets import Save, Screenshot, State from models.platform import Platform from models.rom import Rom, RomFile, RomMetadata, RomUser from sqlalchemy import ( @@ -24,7 +25,7 @@ from sqlalchemy import ( text, update, ) -from sqlalchemy.orm import Query, Session, selectinload +from sqlalchemy.orm import Query, Session, joinedload, noload, selectinload from .base_handler import DBBaseHandler @@ -83,13 +84,25 @@ def with_details(func): @functools.wraps(func) def wrapper(*args, **kwargs): kwargs["query"] = select(Rom).options( - selectinload(Rom.saves), - selectinload(Rom.states), - selectinload(Rom.screenshots), - selectinload(Rom.rom_users), - selectinload(Rom.sibling_roms), - selectinload(Rom.metadatum), - selectinload(Rom.files), + # Ensure platform is loaded for main ROM objects + joinedload(Rom.platform), + selectinload(Rom.saves).options( + noload(Save.rom), + noload(Save.user), + ), + selectinload(Rom.states).options( + noload(State.rom), + noload(State.user), + ), + selectinload(Rom.screenshots).options( + noload(Screenshot.rom), + ), + selectinload(Rom.rom_users).options(noload(RomUser.rom)), + selectinload(Rom.metadatum).options(noload(RomMetadata.rom)), + selectinload(Rom.files).options(noload(RomFile.rom)), + selectinload(Rom.sibling_roms).options( + noload(Rom.platform), noload(Rom.metadatum) + ), selectinload(Rom.collections), ) return func(*args, **kwargs) @@ -101,14 +114,18 @@ def with_simple(func): @functools.wraps(func) def wrapper(*args, **kwargs): kwargs["query"] = select(Rom).options( + # Ensure platform is loaded for main ROM objects + joinedload(Rom.platform), # Display properties for the current user (last_played) - selectinload(Rom.rom_users), + selectinload(Rom.rom_users).options(noload(RomUser.rom)), # Sort table by metadata (first_release_date) - selectinload(Rom.metadatum), + selectinload(Rom.metadatum).options(noload(RomMetadata.rom)), # Required for multi-file ROM actions and 3DS QR code - selectinload(Rom.files), + selectinload(Rom.files).options(noload(RomFile.rom)), # Show sibling rom badges on cards - selectinload(Rom.sibling_roms), + selectinload(Rom.sibling_roms).options( + noload(Rom.platform), noload(Rom.metadatum) + ), ) return func(*args, **kwargs) diff --git a/backend/handler/metadata/playmatch_handler.py b/backend/handler/metadata/playmatch_handler.py index 40e0d834f..05f444483 100644 --- a/backend/handler/metadata/playmatch_handler.py +++ b/backend/handler/metadata/playmatch_handler.py @@ -82,7 +82,7 @@ class PlaymatchHandler: ) res.raise_for_status() return res.json() - except (httpx.HTTPStatusError, httpx.ConnectError) as exc: + except (httpx.HTTPStatusError, httpx.ConnectError, httpx.ReadTimeout) as exc: log.warning("Connection error: can't connect to Playmatch", exc_info=True) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/backend/models/firmware.py b/backend/models/firmware.py index 96eea6df4..489a3456a 100644 --- a/backend/models/firmware.py +++ b/backend/models/firmware.py @@ -42,41 +42,36 @@ class Firmware(BaseModel): crc_hash: Mapped[str] = mapped_column(String(length=100)) md5_hash: Mapped[str] = mapped_column(String(length=100)) sha1_hash: Mapped[str] = mapped_column(String(length=100)) + is_verified: Mapped[bool] = mapped_column(default=False, nullable=False) platform: Mapped[Platform] = relationship(lazy="joined", back_populates="firmware") missing_from_fs: Mapped[bool] = mapped_column(default=False, nullable=False) - @property - def platform_slug(self) -> str: - return self.platform.slug - - @property - def platform_fs_slug(self) -> str: - return self.platform.fs_slug - - @property - def platform_name(self) -> str: - return self.platform.name - - @cached_property - def is_verified(self) -> bool: - cache_entry = sync_cache.hget( - KNOWN_BIOS_KEY, f"{self.platform_slug}:{self.file_name}" - ) - if cache_entry: - cache_json = json.loads(cache_entry) - return self.file_size_bytes == int(cache_json.get("size", 0)) and ( - self.md5_hash == cache_json.get("md5") - or self.sha1_hash == cache_json.get("sha1") - or self.crc_hash == cache_json.get("crc") - ) - - return False - @cached_property def full_path(self) -> str: return f"{self.file_path}/{self.file_name}" + @classmethod + def verify_file_hashes( + cls, + platform_slug: str, + file_name: str, + file_size_bytes: int, + md5_hash: str, + sha1_hash: str, + crc_hash: str, + ) -> bool: + cache_entry = sync_cache.hget(KNOWN_BIOS_KEY, f"{platform_slug}:{file_name}") + if cache_entry: + cache_json = json.loads(cache_entry) + return file_size_bytes == int(cache_json.get("size", 0)) and ( + md5_hash == cache_json.get("md5") + or sha1_hash == cache_json.get("sha1") + or crc_hash == cache_json.get("crc") + ) + + return False + def __repr__(self) -> str: return self.file_name