diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 5dcee1f0b..f5525d069 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -21,14 +21,14 @@ lint: - actionlint@1.7.0 - bandit@1.7.8 - black@24.4.2 - - checkov@3.2.98 + - checkov@3.2.106 - git-diff-check - isort@5.13.2 - mypy@1.10.0 - osv-scanner@1.7.3 - oxipng@9.1.1 - prettier@3.2.5 - - ruff@0.4.4 + - ruff@0.4.5 - shellcheck@0.10.0 - shfmt@3.6.0 - svgo@3.3.2 diff --git a/backend/config/__init__.py b/backend/config/__init__.py index fbdbf3339..886944b79 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -6,10 +6,11 @@ from dotenv import load_dotenv load_dotenv() -# UVICORN +# GUNICORN DEV_PORT: Final = int(os.environ.get("VITE_BACKEND_DEV_PORT", "5000")) DEV_HOST: Final = "127.0.0.1" ROMM_HOST: Final = os.environ.get("ROMM_HOST", DEV_HOST) +GUNICORN_WORKERS: Final = int(os.environ.get("GUNICORN_WORKERS", 2)) # PATHS ROMM_BASE_PATH: Final = os.environ.get("ROMM_BASE_PATH", "/romm") diff --git a/backend/endpoints/platform.py b/backend/endpoints/platform.py index a2c702dbc..542e65bad 100644 --- a/backend/endpoints/platform.py +++ b/backend/endpoints/platform.py @@ -95,9 +95,7 @@ def get_platform(request: Request, id: int) -> PlatformSchema: PlatformSchema: Platform """ - return PlatformSchema.from_orm_with_request( - db_platform_handler.get_platforms(id), request - ) + return db_platform_handler.get_platforms(id) @protected_route(router.put, "/platforms/{id}", ["platforms.write"]) diff --git a/backend/endpoints/responses/platform.py b/backend/endpoints/responses/platform.py index 0fe0bfa5b..5164a343f 100644 --- a/backend/endpoints/responses/platform.py +++ b/backend/endpoints/responses/platform.py @@ -1,7 +1,4 @@ from typing import Optional - -from fastapi import Request -from models.platform import Platform from pydantic import BaseModel, Field from .firmware import FirmwareSchema @@ -17,19 +14,7 @@ class PlatformSchema(BaseModel): name: str logo_path: Optional[str] = "" rom_count: int - - firmware_files: list[FirmwareSchema] = Field(default_factory=list) + firmware: list[FirmwareSchema] = Field(default_factory=list) class Config: from_attributes = True - - @classmethod - def from_orm_with_request( - cls, db_platform: Platform, request: Request - ) -> "PlatformSchema": - platform = cls.model_validate(db_platform) - platform.firmware_files = [ - FirmwareSchema.model_validate(f) - for f in sorted(db_platform.firmware, key=lambda x: x.file_name) - ] - return platform diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 0dcc7d166..f8675ba20 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -23,7 +23,7 @@ from handler.filesystem.base_handler import CoverSize from handler.metadata import meta_igdb_handler, meta_moby_handler from logger.logger import log from stream_zip import ZIP_AUTO, stream_zip # type: ignore[import] - +from urllib.parse import quote router = APIRouter() @@ -236,7 +236,7 @@ def get_rom_content( return CustomStreamingResponse( zipped_chunks, media_type="application/zip", - headers={"Content-Disposition": f'attachment; filename="{file_name}.zip"'}, + headers={"Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"'}, emit_body={"id": rom.id}, ) diff --git a/backend/handler/database/platforms_handler.py b/backend/handler/database/platforms_handler.py index 7934481c3..e3480415b 100644 --- a/backend/handler/database/platforms_handler.py +++ b/backend/handler/database/platforms_handler.py @@ -1,41 +1,58 @@ +import functools +from sqlalchemy import delete, or_ +from sqlalchemy.orm import Session, Query, joinedload + from decorators.database import begin_session from models.platform import Platform from models.rom import Rom -from sqlalchemy import delete, func, or_, select -from sqlalchemy.orm import Session from .base_handler import DBBaseHandler +def with_query(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + session = kwargs.get("session") + if session is None: + raise ValueError("session is required") + + kwargs["query"] = session.query(Platform).options(joinedload(Platform.roms)) + return func(*args, **kwargs) + + return wrapper + + class DBPlatformsHandler(DBBaseHandler): @begin_session - def add_platform(self, platform: Platform, session: Session = None) -> Platform: - return session.merge(platform) + @with_query + def add_platform( + self, platform: Platform, query: Query = None, session: Session = None + ) -> Platform | None: + session.merge(platform) + session.flush() + + return query.filter(Platform.fs_slug == platform.fs_slug).first() @begin_session + @with_query def get_platforms( - self, id: int | None = None, session: Session = None - ) -> Platform | list[Platform] | None: + self, id: int = None, query: Query = None, session: Session = None + ) -> list[Platform] | Platform | None: return ( - session.get(Platform, id) + query.get(id) if id - else ( - session.scalars(select(Platform).order_by(Platform.name.asc())) # type: ignore[attr-defined] - .unique() - .all() - ) + else (session.scalars(query.order_by(Platform.name.asc())).unique().all()) ) @begin_session + @with_query def get_platform_by_fs_slug( - self, fs_slug: str, session: Session = None + self, fs_slug: str, query: Query = None, session: Session = None ) -> Platform | None: - return session.scalars( - select(Platform).filter_by(fs_slug=fs_slug).limit(1) - ).first() + return session.scalars(query.filter_by(fs_slug=fs_slug).limit(1)).first() @begin_session - def delete_platform(self, id: int, session: Session = None) -> None: + def delete_platform(self, id: int, session: Session = None) -> int: # Remove all roms from that platforms first session.execute( delete(Rom) @@ -49,13 +66,7 @@ class DBPlatformsHandler(DBBaseHandler): ) @begin_session - def get_rom_count(self, platform_id: int, session: Session = None) -> int: - return session.scalar( - select(func.count()).select_from(Rom).filter_by(platform_id=platform_id) - ) - - @begin_session - def purge_platforms(self, fs_platforms: list[str], session: Session = None) -> None: + def purge_platforms(self, fs_platforms: list[str], session: Session = None) -> int: return session.execute( delete(Platform) .where(or_(Platform.fs_slug.not_in(fs_platforms), Platform.slug.is_(None))) # type: ignore[attr-defined] diff --git a/backend/handler/metadata/igdb_handler.py b/backend/handler/metadata/igdb_handler.py index ad1846532..7105dfe31 100644 --- a/backend/handler/metadata/igdb_handler.py +++ b/backend/handler/metadata/igdb_handler.py @@ -465,7 +465,8 @@ class IGDBBaseHandler(MetadataHandler): if not IGDB_API_ENABLED: return [] - return [self.get_rom_by_id(igdb_id)] + rom = self.get_rom_by_id(igdb_id) + return [rom] if rom["igdb_id"] else [] @check_twitch_token def get_matched_roms_by_name( diff --git a/backend/handler/metadata/moby_handler.py b/backend/handler/metadata/moby_handler.py index 411373bda..442f5b9a4 100644 --- a/backend/handler/metadata/moby_handler.py +++ b/backend/handler/metadata/moby_handler.py @@ -260,14 +260,14 @@ class MobyGamesHandler(MetadataHandler): def get_rom_by_id(self, moby_id: int) -> MobyGamesRom: if not MOBY_API_ENABLED: - return MobyGamesRom(moby_id=moby_id) + return MobyGamesRom(moby_id=None) url = yarl.URL(self.games_url).with_query(id=moby_id) roms = self._request(str(url)).get("games", []) res = pydash.get(roms, "[0]", None) if not res: - return MobyGamesRom(moby_id=moby_id) + return MobyGamesRom(moby_id=None) rom = { "moby_id": res["game_id"], @@ -285,7 +285,8 @@ class MobyGamesHandler(MetadataHandler): if not MOBY_API_ENABLED: return [] - return [self.get_rom_by_id(moby_id)] + rom = self.get_rom_by_id(moby_id) + return [rom] if rom["moby_id"] else [] def get_matched_roms_by_name( self, search_term: str, platform_moby_id: int diff --git a/backend/models/platform.py b/backend/models/platform.py index 0dc6a7d29..ea46b7aaf 100644 --- a/backend/models/platform.py +++ b/backend/models/platform.py @@ -1,8 +1,9 @@ +from sqlalchemy import Column, Integer, String, select, func +from sqlalchemy.orm import Mapped, relationship, column_property + from models.base import BaseModel from models.firmware import Firmware from models.rom import Rom -from sqlalchemy import Column, Integer, String -from sqlalchemy.orm import Mapped, relationship class Platform(BaseModel): @@ -24,11 +25,10 @@ class Platform(BaseModel): "Firmware", lazy="selectin", back_populates="platform" ) - @property - def rom_count(self) -> int: - from handler.database import db_platform_handler - - return db_platform_handler.get_rom_count(self.id) + # This runs a subquery to get the count of roms for the platform + rom_count = column_property( + select(func.count(Rom.id)).where(Rom.platform_id == id).scalar_subquery() + ) def __repr__(self) -> str: return self.name diff --git a/docker/init_scripts/init b/docker/init_scripts/init index c22f68271..b837c023d 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -35,20 +35,20 @@ error_log() { } # function that runs or main process and creates a corresponding PID file, -start_bin_gunicorn() { - # cleanup potentially leftover socket - rm /tmp/gunicorn.sock -f - # Commands to start our main application and store its PID to check for crashes - info_log "starting gunicorn" - gunicorn \ - --access-logfile - \ - --error-logfile - \ - --worker-class uvicorn.workers.UvicornWorker \ - --bind=0.0.0.0:5000 \ - --bind=unix:/tmp/gunicorn.sock \ - --pid=/tmp/gunicorn.pid \ - --workers 2 \ - main:app & +start_bin_gunicorn () { + # cleanup potentially leftover socket + rm /tmp/gunicorn.sock -f + # Commands to start our main application and store its PID to check for crashes + info_log "starting gunicorn" + gunicorn \ + --access-logfile - \ + --error-logfile - \ + --worker-class uvicorn.workers.UvicornWorker \ + --bind=0.0.0.0:5000 \ + --bind=unix:/tmp/gunicorn.sock \ + --pid=/tmp/gunicorn.pid \ + --workers ${GUNICORN_WORKERS:=2} \ + main:app & } # Commands to start nginx (handling PID creation internally) diff --git a/env.template b/env.template index 8a7701189..92a62393e 100644 --- a/env.template +++ b/env.template @@ -1,6 +1,9 @@ ROMM_BASE_PATH=/path/to/romm_mock VITE_BACKEND_DEV_PORT=5000 + +# Gunicorn (optional) ROMM_HOST=localhost +GUNICORN_WORKERS=4 # (2 × CPU cores) + 1 # IGDB credentials IGDB_CLIENT_ID= diff --git a/frontend/assets/platforms/amiibo.ico b/frontend/assets/platforms/amiibo.ico new file mode 100644 index 000000000..54b481839 Binary files /dev/null and b/frontend/assets/platforms/amiibo.ico differ diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 0241405af..5acbfc9b0 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,17 +1,21 @@