rommapp/romm
1
0
Fork 0
mirror of https://github.com/rommapp/romm.git synced 2026-02-18 00:27:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2303 from rommapp/db-query-micro-optimize

Browse Source 
DB query micro optimizations
This commit is contained in:
Georges-Antoine Assi
2025-08-22 11:38:53 -05:00
committed by GitHub
parent aef721eec4 898f960c96
commit 9fe4eca302
8 changed files with 151 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
backend/alembic/versions/0050_firmware_add_is_verified.py Normal file
View File

@@ -0,0 +1,55 @@
"""Add is_verified column to firmware table
Revision ID: 0050_firmware_add_is_verified
Revises: 0049_add_fs_size_bytes
Create Date: 2025-08-22 04:42:22.367888
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "0050_firmware_add_is_verified"
down_revision = "0049_add_fs_size_bytes"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("firmware", schema=None) as batch_op:
batch_op.add_column(sa.Column("is_verified", sa.Boolean(), nullable=True))
# Calculate is_verified for all firmware
from handler.database import db_firmware_handler
from models.firmware import Firmware
all_firmware = db_firmware_handler.list_firmware()
verified_firmware_ids = []
for firmware in all_firmware:
is_verified = Firmware.verify_file_hashes(
platform_slug=firmware.platform.slug,
file_name=firmware.file_name,
file_size_bytes=firmware.file_size_bytes,
md5_hash=firmware.md5_hash,
sha1_hash=firmware.sha1_hash,
crc_hash=firmware.crc_hash,
)
if is_verified:
verified_firmware_ids.append(firmware.id)
op.execute("UPDATE firmware SET is_verified = 0")
if verified_firmware_ids:
op.execute(
# trunk-ignore(bandit/B608)
f"UPDATE firmware SET is_verified = 1 WHERE id IN ({','.join(map(str, verified_firmware_ids))})"
)
with op.batch_alter_table("firmware", schema=None) as batch_op:
batch_op.alter_column("is_verified", existing_type=sa.Boolean(), nullable=False)
def downgrade() -> None:
with op.batch_alter_table("firmware", schema=None) as batch_op:
batch_op.drop_column("is_verified")

19
backend/endpoints/firmware.py
View File

@@ -11,6 +11,7 @@ from handler.scan_handler import scan_firmware
from logger.formatter import BLUE
from logger.formatter import highlight as hl
from logger.logger import log
from models.firmware import Firmware
from utils.router import APIRouter
router = APIRouter(
@@ -69,13 +70,27 @@ async def add_firmware(
firmware=db_firmware,
)
is_verified = Firmware.verify_file_hashes(
platform_slug=db_platform.slug,
file_name=file.filename,
file_size_bytes=scanned_firmware.file_size_bytes,
md5_hash=scanned_firmware.md5_hash,
sha1_hash=scanned_firmware.sha1_hash,
crc_hash=scanned_firmware.crc_hash,
)
if db_firmware:
db_firmware_handler.update_firmware(
db_firmware.id, {"file_size_bytes": scanned_firmware.file_size_bytes}
db_firmware.id,
{
"file_size_bytes": scanned_firmware.file_size_bytes,
"is_verified": is_verified,
},
)
continue
scanned_firmware.platform_id = db_platform.id
scanned_firmware.is_verified = is_verified
db_firmware_handler.add_firmware(scanned_firmware)
uploaded_firmware.append(scanned_firmware)
@@ -239,7 +254,7 @@ async def delete_firmware(
file_path = f"{firmware.file_path}/{firmware.file_name}"
await fs_firmware_handler.remove_file(file_path=file_path)
except FileNotFoundError:
error = f"Firmware file {hl(firmware.file_name)} not found for platform {hl(firmware.platform_slug)}"
error = f"Firmware file {hl(firmware.file_name)} not found for platform {hl(firmware.platform.slug)}"
log.error(error)
errors.append(error)
failed_items += 1

12
backend/endpoints/sockets/scan.py
View File

@@ -33,6 +33,7 @@ from handler.socket_handler import socket_handler
from logger.formatter import BLUE, LIGHTYELLOW
from logger.formatter import highlight as hl
from logger.logger import log
from models.firmware import Firmware
from models.platform import Platform
from models.rom import Rom, RomFile
from rq import Worker
@@ -95,11 +96,22 @@ async def _identify_firmware(
firmware=firmware,
)
is_verified = Firmware.verify_file_hashes(
platform_slug=platform.slug,
file_name=fs_fw,
file_size_bytes=scanned_firmware.file_size_bytes,
md5_hash=scanned_firmware.md5_hash,
sha1_hash=scanned_firmware.sha1_hash,
crc_hash=scanned_firmware.crc_hash,
)
scan_stats.scanned_firmware += 1
scan_stats.added_firmware += 1 if not firmware else 0
scanned_firmware.missing_from_fs = False
scanned_firmware.is_verified = is_verified
db_firmware_handler.add_firmware(scanned_firmware)
return scan_stats

8
backend/handler/database/base_handler.py
View File

@@ -1,3 +1,4 @@
import logging
import time
from config import DEV_SQL_ECHO
@@ -6,10 +7,13 @@ from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
sync_engine = create_engine(
ConfigManager.get_db_engine(), pool_pre_ping=True, echo=DEV_SQL_ECHO
ConfigManager.get_db_engine(), pool_pre_ping=True, echo=False
)
sync_session = sessionmaker(bind=sync_engine, expire_on_commit=False)
# Disable SQLAlchemy logging as echo will print the queries
logging.getLogger("sqlalchemy.engine.Engine").handlers = [logging.NullHandler()]
if DEV_SQL_ECHO:
@@ -18,6 +22,7 @@ if DEV_SQL_ECHO:
conn, cursor, statement, parameters, context, executemany
):
context._query_start_time = time.time()
print("--------START--------")
print(f"SQL: {statement}")
print(f"Parameters: {parameters}")
@@ -25,6 +30,7 @@ if DEV_SQL_ECHO:
def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
total_time = time.time() - context._query_start_time
print(f"Execution time: {total_time:.4f} seconds")
print("--------END--------")
class DBBaseHandler: ...

10
backend/handler/database/collections_handler.py
View File

@@ -11,7 +11,7 @@ from models.collection import (
)
from models.rom import Rom
from sqlalchemy import delete, insert, literal, or_, select, update
from sqlalchemy.orm import Query, Session, selectinload
from sqlalchemy.orm import Query, Session, noload, selectinload
from .base_handler import DBBaseHandler
@@ -20,7 +20,13 @@ def with_roms(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
kwargs["query"] = select(Collection).options(
selectinload(Collection.roms),
selectinload(Collection.roms)
.load_only(
Rom.id,
Rom.path_cover_s,
Rom.path_cover_l,
)
.options(noload(Rom.platform), noload(Rom.metadatum))
)
return func(*args, **kwargs)

41
backend/handler/database/roms_handler.py
View File

@@ -5,6 +5,7 @@ from typing import Any
from config import ROMM_DB_DRIVER
from decorators.database import begin_session
from handler.metadata.base_hander import UniversalPlatformSlug as UPS
from models.assets import Save, Screenshot, State
from models.platform import Platform
from models.rom import Rom, RomFile, RomMetadata, RomUser
from sqlalchemy import (
@@ -24,7 +25,7 @@ from sqlalchemy import (
text,
update,
)
from sqlalchemy.orm import Query, Session, selectinload
from sqlalchemy.orm import Query, Session, joinedload, noload, selectinload
from .base_handler import DBBaseHandler
@@ -83,13 +84,25 @@ def with_details(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
kwargs["query"] = select(Rom).options(
selectinload(Rom.saves),
selectinload(Rom.states),
selectinload(Rom.screenshots),
selectinload(Rom.rom_users),
selectinload(Rom.sibling_roms),
selectinload(Rom.metadatum),
selectinload(Rom.files),
# Ensure platform is loaded for main ROM objects
joinedload(Rom.platform),
selectinload(Rom.saves).options(
noload(Save.rom),
noload(Save.user),
),
selectinload(Rom.states).options(
noload(State.rom),
noload(State.user),
),
selectinload(Rom.screenshots).options(
noload(Screenshot.rom),
),
selectinload(Rom.rom_users).options(noload(RomUser.rom)),
selectinload(Rom.metadatum).options(noload(RomMetadata.rom)),
selectinload(Rom.files).options(noload(RomFile.rom)),
selectinload(Rom.sibling_roms).options(
noload(Rom.platform), noload(Rom.metadatum)
),
selectinload(Rom.collections),
)
return func(*args, **kwargs)
@@ -101,14 +114,18 @@ def with_simple(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
kwargs["query"] = select(Rom).options(
# Ensure platform is loaded for main ROM objects
joinedload(Rom.platform),
# Display properties for the current user (last_played)
selectinload(Rom.rom_users),
selectinload(Rom.rom_users).options(noload(RomUser.rom)),
# Sort table by metadata (first_release_date)
selectinload(Rom.metadatum),
selectinload(Rom.metadatum).options(noload(RomMetadata.rom)),
# Required for multi-file ROM actions and 3DS QR code
selectinload(Rom.files),
selectinload(Rom.files).options(noload(RomFile.rom)),
# Show sibling rom badges on cards
selectinload(Rom.sibling_roms),
selectinload(Rom.sibling_roms).options(
noload(Rom.platform), noload(Rom.metadatum)
),
)
return func(*args, **kwargs)

2
backend/handler/metadata/playmatch_handler.py
View File

@@ -82,7 +82,7 @@ class PlaymatchHandler:
)
res.raise_for_status()
return res.json()
except (httpx.HTTPStatusError, httpx.ConnectError) as exc:
except (httpx.HTTPStatusError, httpx.ConnectError, httpx.ReadTimeout) as exc:
log.warning("Connection error: can't connect to Playmatch", exc_info=True)
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,

49
backend/models/firmware.py
View File

@@ -42,41 +42,36 @@ class Firmware(BaseModel):
crc_hash: Mapped[str] = mapped_column(String(length=100))
md5_hash: Mapped[str] = mapped_column(String(length=100))
sha1_hash: Mapped[str] = mapped_column(String(length=100))
is_verified: Mapped[bool] = mapped_column(default=False, nullable=False)
platform: Mapped[Platform] = relationship(lazy="joined", back_populates="firmware")
missing_from_fs: Mapped[bool] = mapped_column(default=False, nullable=False)
@property
def platform_slug(self) -> str:
return self.platform.slug
@property
def platform_fs_slug(self) -> str:
return self.platform.fs_slug
@property
def platform_name(self) -> str:
return self.platform.name
@cached_property
def is_verified(self) -> bool:
cache_entry = sync_cache.hget(
KNOWN_BIOS_KEY, f"{self.platform_slug}:{self.file_name}"
)
if cache_entry:
cache_json = json.loads(cache_entry)
return self.file_size_bytes == int(cache_json.get("size", 0)) and (
self.md5_hash == cache_json.get("md5")
or self.sha1_hash == cache_json.get("sha1")
or self.crc_hash == cache_json.get("crc")
)
return False
@cached_property
def full_path(self) -> str:
return f"{self.file_path}/{self.file_name}"
@classmethod
def verify_file_hashes(
cls,
platform_slug: str,
file_name: str,
file_size_bytes: int,
md5_hash: str,
sha1_hash: str,
crc_hash: str,
) -> bool:
cache_entry = sync_cache.hget(KNOWN_BIOS_KEY, f"{platform_slug}:{file_name}")
if cache_entry:
cache_json = json.loads(cache_entry)
return file_size_bytes == int(cache_json.get("size", 0)) and (
md5_hash == cache_json.get("md5")
or sha1_hash == cache_json.get("sha1")
or crc_hash == cache_json.get("crc")
)
return False
def __repr__(self) -> str:
return self.file_name