From 5649ebfc030dcd30cd528776e5301e7f290f11a0 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Fri, 30 Aug 2024 17:20:27 -0400 Subject: [PATCH 01/24] Add bios files for doom --- backend/models/fixtures/known_bios_files.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/backend/models/fixtures/known_bios_files.json b/backend/models/fixtures/known_bios_files.json index 34e844225..a3ae4e787 100644 --- a/backend/models/fixtures/known_bios_files.json +++ b/backend/models/fixtures/known_bios_files.json @@ -2122,5 +2122,95 @@ "crc": "1466aed4", "md5": "88dc7876d584f90e4106f91444ab23b7", "sha1": "072c6160d4e7d406f5d8f5b1b66066c797d35561" + }, + "doom:DOOM1.WAD": { + "size": "4196020", + "crc": "162b696a", + "md5": "f0cefca49926d00903cf57551d901abe", + "sha1": "5b2e249b9c5133ec987b3ea77596381dc0d6bc1d" + }, + "doom:DOOM.WAD": { + "size": "12733492", + "crc": "cff03d9f", + "md5": "4461d4511386518e784c647e3128e7bc", + "sha1": "997bae5e5a190c5bb3b1fb9e7e3e75b2da88cb27" + }, + "doom:DOOM2.WAD": { + "size": "14802506", + "crc": "09b8a6ae", + "md5": "9aa3cbf65b961d0bdac98ec403b832e1", + "sha1": "c745f04a6abc2e6d2a2d52382f45500dd2a260be" + }, + "doom:DOOM2F.WAD": { + "size": "14607420", + "crc": "27eaae69", + "md5": "3cb02349b3df649c86290907eed64e7b", + "sha1": "d510c877031bbd5f3d198581a2c8651e09b9861f" + }, + "doom:DOOM64.WAD": { + "size": "15103212", + "crc": "65816192", + "md5": "0aaba212339c72250f8a53a0a2b6189e", + "sha1": "d041456bea851c173f65ac6ab3f2ee61bb0b8b53" + }, + "doom:TNT.WAD": { + "size": "18304630", + "crc": "15f18ddb", + "md5": "8974e3117ed4a1839c752d5e11ab1b7b", + "sha1": "9820e2a3035f0cdd87f69a7d57c59a7a267c9409" + }, + "doom:PLUTONIA.WAD": { + "size": "17531493", + "crc": "650b998d", + "md5": "24037397056e919961005e08611623f4", + "sha1": " 816c7c6b0098f66c299c9253f62bd908456efb63" + }, + "doom:HERETIC1.WAD": { + "size": "5120920", + "crc": "22d3f0ca", + "md5": "ae779722390ec32fa37b0d361f7d82f8", + "sha1": "b4c50ca9bea07f7c35250a1a11906091971c05ae" + }, + "doom:HERETIC.WAD": { + "size": "14189976", + "crc": "5b16049e", + "md5": "66d686b1ed6d35ff103f15dbd30e0341", + "sha1": "f489d479371df32f6d280a0cb23b59a35ba2b833" + }, + "doom:HEXEN.WAD": { + "size": "20083672", + "crc": "dca9114c", + "md5": "abb033caf81e26f12a2103e1fa25453f", + "sha1": "4b53832f0733c1e29e5f1de2428e5475e891af29" + }, + "doom:HEXDD.WAD": { + "size": "4440584", + "crc": "fd5eb11d", + "md5": "78d5898e99e220e4de64edaa0e479593", + "sha1": "081f6a2024643b54ef4a436a85508539b6d20a1e" + }, + "doom:STRIFE0.WAD": { + "size": "9934413", + "crc": "93c144dd", + "md5": "bb545b9c4eca0ff92c14d466b3294023", + "sha1": "bc0a110bf27aee89a0b2fc8111e2391ede891b8d" + }, + "doom:STRIFE1.WAD": { + "size": "28377364", + "crc": "4234ace5", + "md5": "2fed2031a5b03892106e0f117f17901f", + "sha1": "64c13b951a845ca7f8081f68138a6181557458d1" + }, + "doom:VOICES.WAD": { + "size": "27319149", + "crc": "cd12ebcf", + "md5": "082234d6a3f7086424856478b5aa9e95", + "sha1": "ec6883100d807b894a98f426d024d22c77b63e7f" + }, + "doom:CHEX.WAD": { + "size": "12361532", + "crc": "298dd5b5", + "md5": "25485721882b050afa96a56e5758dd52", + "sha1": "eca9cff1014ce5081804e193588d96c6ddb35432" } } From 962f5835afdfae5307d0414164db1cd37a4ff685 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 31 Aug 2024 09:49:58 -0400 Subject: [PATCH 02/24] add 4 more from retrodeck --- backend/models/fixtures/known_bios_files.json | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/backend/models/fixtures/known_bios_files.json b/backend/models/fixtures/known_bios_files.json index a3ae4e787..8d9877009 100644 --- a/backend/models/fixtures/known_bios_files.json +++ b/backend/models/fixtures/known_bios_files.json @@ -173,6 +173,12 @@ "md5": "79ae0d2bb1901b7e606b6dc339b79a97", "sha1": "06fc753d015b43ca1787f4cfd9331b1674202e64" }, + "arcade:naomi2.zip": { + "size": "2189528", + "crc": "f25cf3a8", + "md5": "0ea5bf0345e27b1cf51bbde1bd398eca", + "sha1": "533ef12a6d22726da81a50f08871c6e9a377a328" + }, "atari8bit:ATARIBAS.ROM": { "size": "8192", "crc": "7d684184", @@ -335,6 +341,12 @@ "md5": "80dcd1ad1a4cf65d64b7ba10504e8190", "sha1": "032cb1c1c75b9a191fa1230978971698d9d2a17f" }, + "msx2:DISK.ROM": { + "size": "16384", + "crc": "721f61df", + "md5": "80dcd1ad1a4cf65d64b7ba10504e8190", + "sha1": "032cb1c1c75b9a191fa1230978971698d9d2a17f" + }, "msx:FMPAC.ROM": { "size": "65536", "crc": "0e84505d", @@ -1439,6 +1451,12 @@ "md5": "08ca8b2dba6662e8024f9e789711c6fc", "sha1": "5142f205912869b673a71480c5828b1eaed782a8" }, + "neo-geo-cd:neocdz.zip": { + "size": "214876", + "crc": "19681e91", + "md5": "c38cb8e50321783e413dc5ff292a3ff8", + "sha1": "bf6b379c204da77dece1aedf83ff35227a623e5d" + }, "ps:PSXONPSP660.BIN": { "size": "524288", "crc": "5660f34f", @@ -2212,5 +2230,17 @@ "crc": "298dd5b5", "md5": "25485721882b050afa96a56e5758dd52", "sha1": "eca9cff1014ce5081804e193588d96c6ddb35432" + }, + "xbox:Complex_4627.bin": { + "size": "1048576", + "crc": "ccb97a84", + "md5": "ec00e31e746de2473acfe7903c5a4cb7", + "sha1": "6639b6693784574d204c42703a74fd8b088a3a5e" + }, + "xbox:mcpx_1.0.bin": { + "size": "512", + "crc": "0b07d1f1", + "md5": "d49c52a4102f6df7bcf8d0617ac475ed", + "sha1": "5d270675b54eb8071b480e42d22a3015ac211cef" } } From 12c27a1e5e7c44cff244bfe2d4f03725945241ae Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Mon, 27 Jan 2025 10:38:18 -0500 Subject: [PATCH 03/24] migration to generate virtual collections --- .../0034_virtual_collections_db_view.py | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 backend/alembic/versions/0034_virtual_collections_db_view.py diff --git a/backend/alembic/versions/0034_virtual_collections_db_view.py b/backend/alembic/versions/0034_virtual_collections_db_view.py new file mode 100644 index 000000000..9a26a5c1a --- /dev/null +++ b/backend/alembic/versions/0034_virtual_collections_db_view.py @@ -0,0 +1,196 @@ +"""empty message + +Revision ID: 0034_virtual_collections_db_view +Revises: 0033_rom_file_and_hashes +Create Date: 2024-08-08 12:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op +from utils.database import is_postgresql + +# revision identifiers, used by Alembic. +revision = "0034_virtual_collections_db_view" +down_revision = "0033_rom_file_and_hashes" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + connection = op.get_bind() + if is_postgresql(connection): + connection.execute( + sa.text( + """ + CREATE OR REPLACE VIEW virtual_collections AS + WITH genres_collection AS ( + SELECT + r.id as rom_id, + jsonb_array_elements_text(to_jsonb(igdb_metadata ->> 'genres')) as collection_name, + 'genre' as collection_type + FROM roms r + WHERE igdb_metadata->>'genres' IS NOT NULL + ), + franchises_collection AS ( + SELECT + r.id as rom_id, + jsonb_array_elements_text(to_jsonb(igdb_metadata->>'franchises')) as collection_name, + 'franchise' as collection_type + FROM roms r + WHERE igdb_metadata->>'franchises' IS NOT NULL + ), + collection_collection AS ( + SELECT + r.id as rom_id, + jsonb_array_elements_text(to_jsonb(igdb_metadata->>'collections')) as collection_name, + 'collection' as collection_type + FROM roms r + WHERE igdb_metadata->>'collections' IS NOT NULL + ), + modes_collection AS ( + SELECT + r.id as rom_id, + jsonb_array_elements_text(to_jsonb(igdb_metadata->>'game_modes')) as collection_name, + 'mode' as collection_type + FROM roms r + WHERE igdb_metadata->>'game_modes' IS NOT NULL + ), + companies_collection AS ( + SELECT + r.id as rom_id, + jsonb_array_elements_text(to_jsonb(igdb_metadata->>'companies')) as collection_name, + 'company' as collection_type + FROM roms r + WHERE igdb_metadata->>'companies' IS NOT NULL + ) + SELECT + collection_name, + collection_type, + array_agg(DISTINCT rom_id) as roms + FROM ( + SELECT * FROM genres_collection + UNION ALL + SELECT * FROM franchises_collection + UNION ALL + SELECT * FROM collection_collection + UNION ALL + SELECT * FROM modes_collection + UNION ALL + SELECT * FROM companies_collection + ) combined + GROUP BY collection_type, collection_name + HAVING COUNT(DISTINCT rom_id) > 1 + ORDER BY collection_type, collection_name; + """ # nosec B608 + ), + ) + else: + connection.execute( + sa.text( + """ + CREATE OR REPLACE VIEW virtual_collections AS + WITH genres AS ( + SELECT + r.id as rom_id, + CONCAT(j.genre) as collection_name, + 'genre' as collection_type + FROM + roms r + CROSS JOIN JSON_TABLE( + JSON_EXTRACT(igdb_metadata, '$.genres'), + '$[*]' COLUMNS (genre VARCHAR(255) PATH '$') + ) j + WHERE + JSON_EXTRACT(igdb_metadata, '$.genres') IS NOT NULL + ), + franchises AS ( + SELECT + r.id as rom_id, + CONCAT(j.franchise) as collection_name, + 'franchise' as collection_type + FROM + roms r + CROSS JOIN JSON_TABLE( + JSON_EXTRACT(igdb_metadata, '$.franchises'), + '$[*]' COLUMNS (franchise VARCHAR(255) PATH '$') + ) j + WHERE + JSON_EXTRACT(igdb_metadata, '$.franchises') IS NOT NULL + ), + collections AS ( + SELECT + r.id as rom_id, + CONCAT(j.collection) as collection_name, + 'collection' as collection_type + FROM + roms r + CROSS JOIN JSON_TABLE( + JSON_EXTRACT(igdb_metadata, '$.collections'), + '$[*]' COLUMNS (collection VARCHAR(255) PATH '$') + ) j + WHERE + JSON_EXTRACT(igdb_metadata, '$.collections') IS NOT NULL + ), + modes AS ( + SELECT + r.id as rom_id, + CONCAT(j.mode) as collection_name, + 'mode' as collection_type + FROM + roms r + CROSS JOIN JSON_TABLE( + JSON_EXTRACT(igdb_metadata, '$.game_modes'), + '$[*]' COLUMNS (mode VARCHAR(255) PATH '$') + ) j + WHERE + JSON_EXTRACT(igdb_metadata, '$.game_modes') IS NOT NULL + ), + companies AS ( + SELECT + r.id as rom_id, + CONCAT(j.company) as collection_name, + 'company' as collection_type + FROM + roms r + CROSS JOIN JSON_TABLE( + JSON_EXTRACT(igdb_metadata, '$.companies'), + '$[*]' COLUMNS (company VARCHAR(255) PATH '$') + ) j + WHERE + JSON_EXTRACT(igdb_metadata, '$.companies') IS NOT NULL + ) + SELECT + collection_name, + collection_type, + GROUP_CONCAT(DISTINCT rom_id) as roms + FROM + ( + SELECT * FROM genres + UNION ALL + SELECT * FROM franchises + UNION ALL + SELECT * FROM collections + UNION ALL + SELECT * FROM modes + UNION ALL + SELECT * FROM companies + ) combined + GROUP BY collection_type, collection_name + HAVING COUNT(DISTINCT rom_id) > 1 + ORDER BY collection_type, collection_name; + """ + ), + ) + + +def downgrade() -> None: + connection = op.get_bind() + + connection.execute( + sa.text( + """ + DROP VIEW virtual_collections; + """ + ), + ) From dbe4b71c7a0b4253d687f6f6cd3ab74231639694 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Mon, 27 Jan 2025 18:15:52 -0500 Subject: [PATCH 04/24] super naive appraoch to loading virtual collections --- .../0034_virtual_collections_db_view.py | 18 ++++---- backend/endpoints/collections.py | 45 +++++++++++++++++-- backend/endpoints/responses/collection.py | 12 +++++ backend/endpoints/responses/rom.py | 7 ++- .../handler/database/collections_handler.py | 39 +++++++++++++++- backend/models/collection.py | 45 ++++++++++++++++--- backend/models/rom.py | 10 ++++- frontend/src/__generated__/index.ts | 1 + .../__generated__/models/DetailedRomSchema.ts | 2 + .../models/VirtualCollectionSchema.ts | 18 ++++++++ frontend/src/stores/collections.ts | 6 ++- 11 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 frontend/src/__generated__/models/VirtualCollectionSchema.ts diff --git a/backend/alembic/versions/0034_virtual_collections_db_view.py b/backend/alembic/versions/0034_virtual_collections_db_view.py index 9a26a5c1a..1af63ef10 100644 --- a/backend/alembic/versions/0034_virtual_collections_db_view.py +++ b/backend/alembic/versions/0034_virtual_collections_db_view.py @@ -65,9 +65,10 @@ def upgrade() -> None: WHERE igdb_metadata->>'companies' IS NOT NULL ) SELECT - collection_name, - collection_type, - array_agg(DISTINCT rom_id) as roms + collection_name as name, + collection_type as type, + 'Virtual collection of ' || collection_type || ' ' || collection_name AS description, + array_to_json(array_agg(DISTINCT rom_id)) as roms FROM ( SELECT * FROM genres_collection UNION ALL @@ -80,7 +81,7 @@ def upgrade() -> None: SELECT * FROM companies_collection ) combined GROUP BY collection_type, collection_name - HAVING COUNT(DISTINCT rom_id) > 1 + HAVING COUNT(DISTINCT rom_id) > 2 ORDER BY collection_type, collection_name; """ # nosec B608 ), @@ -161,9 +162,10 @@ def upgrade() -> None: JSON_EXTRACT(igdb_metadata, '$.companies') IS NOT NULL ) SELECT - collection_name, - collection_type, - GROUP_CONCAT(DISTINCT rom_id) as roms + collection_name as name, + collection_type as type, + CONCAT('Virtual collection of ', collection_type, ' ', collection_name) AS description, + JSON_ARRAYAGG(DISTINCT rom_id) as roms FROM ( SELECT * FROM genres @@ -177,7 +179,7 @@ def upgrade() -> None: SELECT * FROM companies ) combined GROUP BY collection_type, collection_name - HAVING COUNT(DISTINCT rom_id) > 1 + HAVING COUNT(DISTINCT rom_id) > 2 ORDER BY collection_type, collection_name; """ ), diff --git a/backend/endpoints/collections.py b/backend/endpoints/collections.py index 849361844..f8f371c9f 100644 --- a/backend/endpoints/collections.py +++ b/backend/endpoints/collections.py @@ -6,7 +6,7 @@ from anyio import Path from config import RESOURCES_BASE_PATH from decorators.auth import protected_route from endpoints.responses import MessageResponse -from endpoints.responses.collection import CollectionSchema +from endpoints.responses.collection import CollectionSchema, VirtualCollectionSchema from exceptions.endpoint_exceptions import ( CollectionAlreadyExistsException, CollectionNotFoundInDatabaseException, @@ -96,7 +96,9 @@ async def add_collection( @protected_route(router.get, "/collections", [Scope.COLLECTIONS_READ]) -def get_collections(request: Request) -> list[CollectionSchema]: +def get_collections( + request: Request, +) -> list[CollectionSchema]: """Get collections endpoint Args: @@ -108,9 +110,28 @@ def get_collections(request: Request) -> list[CollectionSchema]: """ collections = db_collection_handler.get_collections() + return CollectionSchema.for_user(request.user.id, [c for c in collections]) +@protected_route(router.get, "/collections/virtual", [Scope.COLLECTIONS_READ]) +def get_virtual_collections( + request: Request, +) -> list[VirtualCollectionSchema]: + """Get virtual collections endpoint + + Args: + request (Request): Fastapi Request object + + Returns: + list[VirtualCollectionSchema]: List of virtual collections + """ + + virtual_collections = db_collection_handler.get_virtual_collections() + + return [VirtualCollectionSchema.model_validate(vc) for vc in virtual_collections] + + @protected_route(router.get, "/collections/{id}", [Scope.COLLECTIONS_READ]) def get_collection(request: Request, id: int) -> CollectionSchema: """Get collections endpoint @@ -124,13 +145,31 @@ def get_collection(request: Request, id: int) -> CollectionSchema: """ collection = db_collection_handler.get_collection(id) - if not collection: raise CollectionNotFoundInDatabaseException(id) return CollectionSchema.model_validate(collection) +@protected_route(router.get, "/collections/virtual/{id}", [Scope.COLLECTIONS_READ]) +def get_virtual_collection(request: Request, id: str) -> VirtualCollectionSchema: + """Get virtual collections endpoint + + Args: + request (Request): Fastapi Request object + id (str): Virtual collection id + + Returns: + VirtualCollectionSchema: Virtual collection + """ + + virtual_collection = db_collection_handler.get_virtual_collection(id) + if not virtual_collection: + raise CollectionNotFoundInDatabaseException(id) + + return VirtualCollectionSchema.model_validate(virtual_collection) + + @protected_route(router.put, "/collections/{id}", [Scope.COLLECTIONS_WRITE]) async def update_collection( request: Request, diff --git a/backend/endpoints/responses/collection.py b/backend/endpoints/responses/collection.py index da8d6ee70..ba95632b3 100644 --- a/backend/endpoints/responses/collection.py +++ b/backend/endpoints/responses/collection.py @@ -34,3 +34,15 @@ class CollectionSchema(BaseModel): for c in collections if c.user_id == user_id or c.is_public ] + + +class VirtualCollectionSchema(BaseModel): + id: str + name: str + type: str + description: str + roms: set[int] + rom_count: int + + class Config: + from_attributes = True diff --git a/backend/endpoints/responses/rom.py b/backend/endpoints/responses/rom.py index 2a6e23f9f..5672a4ae3 100644 --- a/backend/endpoints/responses/rom.py +++ b/backend/endpoints/responses/rom.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from typing import NotRequired, TypedDict, get_type_hints from endpoints.responses.assets import SaveSchema, ScreenshotSchema, StateSchema -from endpoints.responses.collection import CollectionSchema +from endpoints.responses.collection import CollectionSchema, VirtualCollectionSchema from fastapi import Request from handler.metadata.igdb_handler import IGDBMetadata from handler.metadata.moby_handler import MobyMetadata @@ -214,6 +214,7 @@ class DetailedRomSchema(RomSchema): user_screenshots: list[ScreenshotSchema] user_notes: list[UserNotesSchema] user_collections: list[CollectionSchema] + virtual_collections: list[VirtualCollectionSchema] @classmethod def from_orm_with_request(cls, db_rom: Rom, request: Request) -> DetailedRomSchema: @@ -235,6 +236,10 @@ class DetailedRomSchema(RomSchema): db_rom.user_collections = CollectionSchema.for_user( # type: ignore user_id, [c for c in db_rom.get_collections()] ) + db_rom.virtual_collections = [ # type: ignore + VirtualCollectionSchema.model_validate(v) + for v in db_rom.get_virtual_collections() + ] return cls.model_validate(db_rom) diff --git a/backend/handler/database/collections_handler.py b/backend/handler/database/collections_handler.py index b6de43094..be729e8a7 100644 --- a/backend/handler/database/collections_handler.py +++ b/backend/handler/database/collections_handler.py @@ -1,7 +1,7 @@ from typing import Any, Sequence from decorators.database import begin_session -from models.collection import Collection +from models.collection import Collection, VirtualCollection from sqlalchemy import delete, select, update from sqlalchemy.orm import Session from sqlalchemy.sql import ColumnExpressionArgument @@ -24,6 +24,15 @@ class DBCollectionsHandler(DBBaseHandler): def get_collection(self, id: int, session: Session = None) -> Collection | None: return session.scalar(select(Collection).filter_by(id=id).limit(1)) + @begin_session + def get_virtual_collection( + self, id: str, session: Session = None + ) -> VirtualCollection | None: + name, type = VirtualCollection.from_id(id) + return session.scalar( + select(VirtualCollection).filter_by(name=name, type=type).limit(1) + ) + @begin_session def get_collection_by_name( self, name: str, user_id: int, session: Session = None @@ -40,6 +49,18 @@ class DBCollectionsHandler(DBBaseHandler): .all() ) + @begin_session + def get_virtual_collections( + self, session: Session = None + ) -> Sequence[VirtualCollection]: + return ( + session.scalars( + select(VirtualCollection).order_by(VirtualCollection.name.asc()) + ) + .unique() + .all() + ) + @begin_session def get_collections_by_rom_id( self, @@ -56,6 +77,22 @@ class DBCollectionsHandler(DBBaseHandler): return session.scalars(query).all() + @begin_session + def get_virtual_collections_by_rom_id( + self, + rom_id: int, + *, + order_by: Sequence[str | ColumnExpressionArgument[Any]] | None = None, + session: Session = None, + ) -> Sequence[VirtualCollection]: + query = select(VirtualCollection).filter( + json_array_contains_value(VirtualCollection.roms, rom_id, session=session) + ) + if order_by is not None: + query = query.order_by(*order_by) + + return session.scalars(query).all() + @begin_session def update_collection( self, id: int, data: dict, session: Session = None diff --git a/backend/models/collection.py b/backend/models/collection.py index 2de5e96bd..cca095189 100644 --- a/backend/models/collection.py +++ b/backend/models/collection.py @@ -1,10 +1,12 @@ from __future__ import annotations +import base64 +import json from functools import cached_property from models.base import BaseModel from models.user import User -from sqlalchemy import ForeignKey, String, Text +from sqlalchemy import ForeignKey, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from utils.database import CustomJSON @@ -16,20 +18,18 @@ class Collection(BaseModel): name: Mapped[str] = mapped_column(String(length=400)) description: Mapped[str | None] = mapped_column(Text) - + is_public: Mapped[bool] = mapped_column(default=False) path_cover_l: Mapped[str | None] = mapped_column(Text, default="") path_cover_s: Mapped[str | None] = mapped_column(Text, default="") - url_cover: Mapped[str | None] = mapped_column( Text, default="", doc="URL to cover image stored in IGDB" ) roms: Mapped[set[int]] = mapped_column( - CustomJSON(), default=[], doc="Rom id's that belong to this collection" + CustomJSON(), default=[], doc="Rom IDs that belong to this collection" ) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) - is_public: Mapped[bool] = mapped_column(default=False) user: Mapped[User] = relationship(lazy="joined", back_populates="collections") @property @@ -50,3 +50,38 @@ class Collection(BaseModel): def __repr__(self) -> str: return self.name + + +class VirtualCollection(BaseModel): + __tablename__ = "virtual_collections" + + name: Mapped[str] = mapped_column(String(length=400), primary_key=True) + type: Mapped[str] = mapped_column(String(length=50), primary_key=True) + description: Mapped[str | None] = mapped_column(Text) + + roms: Mapped[set[int]] = mapped_column( + CustomJSON(), default=[], doc="Rom IDs that belong to this collection" + ) + + @property + def id(self) -> str: + # Create a reversible encoded ID + data = json.dumps({"name": self.name, "type": self.type}) + return base64.urlsafe_b64encode(data.encode()).decode() + + @classmethod + def from_id(cls, id_: str): + data = json.loads(base64.urlsafe_b64decode(id_).decode()) + return data["name"], data["type"] + + @property + def rom_count(self) -> int: + return len(self.roms) + + __table_args__ = ( + UniqueConstraint( + "name", + "type", + name="unique_virtual_collection_name_type", + ), + ) diff --git a/backend/models/rom.py b/backend/models/rom.py index aa9ea83d0..b3fc3a554 100644 --- a/backend/models/rom.py +++ b/backend/models/rom.py @@ -24,7 +24,7 @@ from utils.database import CustomJSON, safe_float, safe_int if TYPE_CHECKING: from models.assets import Save, Screenshot, State - from models.collection import Collection + from models.collection import Collection, VirtualCollection from models.platform import Platform from models.user import User @@ -183,6 +183,14 @@ class Rom(BaseModel): order_by=[func.lower("name")], ) + def get_virtual_collections(self) -> Sequence[VirtualCollection]: + from handler.database import db_collection_handler + + return db_collection_handler.get_virtual_collections_by_rom_id( + self.id, + order_by=[func.lower("name")], + ) + # Metadata fields @property def youtube_video_id(self) -> str: diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index dfb688fa2..bd89869a1 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -58,6 +58,7 @@ export type { UploadedStatesResponse } from './models/UploadedStatesResponse'; export type { UserNotesSchema } from './models/UserNotesSchema'; export type { UserSchema } from './models/UserSchema'; export type { ValidationError } from './models/ValidationError'; +export type { VirtualCollectionSchema } from './models/VirtualCollectionSchema'; export type { WatcherDict } from './models/WatcherDict'; export type { WebrcadeFeedCategorySchema } from './models/WebrcadeFeedCategorySchema'; export type { WebrcadeFeedItemPropsSchema } from './models/WebrcadeFeedItemPropsSchema'; diff --git a/frontend/src/__generated__/models/DetailedRomSchema.ts b/frontend/src/__generated__/models/DetailedRomSchema.ts index a6aa7fae2..aa1a3203d 100644 --- a/frontend/src/__generated__/models/DetailedRomSchema.ts +++ b/frontend/src/__generated__/models/DetailedRomSchema.ts @@ -12,6 +12,7 @@ import type { SaveSchema } from './SaveSchema'; import type { ScreenshotSchema } from './ScreenshotSchema'; import type { StateSchema } from './StateSchema'; import type { UserNotesSchema } from './UserNotesSchema'; +import type { VirtualCollectionSchema } from './VirtualCollectionSchema'; export type DetailedRomSchema = { id: number; igdb_id: (number | null); @@ -68,6 +69,7 @@ export type DetailedRomSchema = { user_screenshots: Array; user_notes: Array; user_collections: Array; + virtual_collections: Array; readonly sort_comparator: string; }; diff --git a/frontend/src/__generated__/models/VirtualCollectionSchema.ts b/frontend/src/__generated__/models/VirtualCollectionSchema.ts new file mode 100644 index 000000000..cff8ab644 --- /dev/null +++ b/frontend/src/__generated__/models/VirtualCollectionSchema.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type VirtualCollectionSchema = { + id: string; + description: string; + name: string; + type: string; + roms: Array; + user_id: number; + is_public: boolean; + rom_count: number; + has_cover: boolean; + created_at: string; + updated_at: string; +}; + diff --git a/frontend/src/stores/collections.ts b/frontend/src/stores/collections.ts index 205af8baf..ee7a5c22d 100644 --- a/frontend/src/stores/collections.ts +++ b/frontend/src/stores/collections.ts @@ -1,9 +1,13 @@ -import type { CollectionSchema } from "@/__generated__"; +import type { + CollectionSchema, + VirtualCollectionSchema, +} from "@/__generated__"; import { uniqBy } from "lodash"; import { defineStore } from "pinia"; import type { SimpleRom } from "./roms"; export type Collection = CollectionSchema; +export type VirtualCollection = VirtualCollectionSchema; export default defineStore("collections", { state: () => { From e2c0a5f54d49428a0e16c453b8f255202eed6d5d Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Mon, 27 Jan 2025 18:43:01 -0500 Subject: [PATCH 05/24] cleanup path covers --- backend/endpoints/responses/collection.py | 6 ++-- backend/endpoints/responses/rom.py | 6 ++-- backend/models/collection.py | 26 ++++++++++++--- backend/models/rom.py | 32 ++++++++++++++----- .../__generated__/models/CollectionSchema.ts | 6 ++-- .../__generated__/models/DetailedRomSchema.ts | 6 ++-- .../src/__generated__/models/RomSchema.ts | 6 ++-- .../__generated__/models/SimpleRomSchema.ts | 6 ++-- .../models/VirtualCollectionSchema.ts | 7 +--- .../components/Details/BackgroundHeader.vue | 5 ++- .../src/components/common/Collection/Card.vue | 22 ++++--------- .../components/common/Collection/RAvatar.vue | 7 ++-- .../src/components/common/Game/Card/Base.vue | 31 ++++++++---------- .../src/components/common/Game/RAvatar.vue | 20 ++++++++---- 14 files changed, 101 insertions(+), 85 deletions(-) diff --git a/backend/endpoints/responses/collection.py b/backend/endpoints/responses/collection.py index ba95632b3..e86193bba 100644 --- a/backend/endpoints/responses/collection.py +++ b/backend/endpoints/responses/collection.py @@ -9,15 +9,15 @@ class CollectionSchema(BaseModel): id: int name: str description: str - path_cover_l: str | None - path_cover_s: str | None - has_cover: bool + path_cover_small: str | None + path_cover_large: str | None url_cover: str roms: set[int] rom_count: int user_id: int user__username: str is_public: bool + is_favorite: bool created_at: datetime updated_at: datetime diff --git a/backend/endpoints/responses/rom.py b/backend/endpoints/responses/rom.py index 5672a4ae3..d0ddaa595 100644 --- a/backend/endpoints/responses/rom.py +++ b/backend/endpoints/responses/rom.py @@ -153,10 +153,10 @@ class RomSchema(BaseModel): igdb_metadata: RomIGDBMetadata | None moby_metadata: RomMobyMetadata | None - path_cover_s: str | None - path_cover_l: str | None - has_cover: bool + path_cover_small: str | None + path_cover_large: str | None url_cover: str | None + is_unidentified: bool revision: str | None regions: list[str] diff --git a/backend/models/collection.py b/backend/models/collection.py index cca095189..b5f5ccd02 100644 --- a/backend/models/collection.py +++ b/backend/models/collection.py @@ -2,8 +2,8 @@ from __future__ import annotations import base64 import json -from functools import cached_property +from config import FRONTEND_RESOURCES_PATH from models.base import BaseModel from models.user import User from sqlalchemy import ForeignKey, String, Text, UniqueConstraint @@ -40,14 +40,30 @@ class Collection(BaseModel): def rom_count(self) -> int: return len(self.roms) - @cached_property - def has_cover(self) -> bool: - return bool(self.path_cover_s or self.path_cover_l) - @property def fs_resources_path(self) -> str: return f"collections/{str(self.id)}" + @property + def path_cover_small(self) -> str: + return ( + f"{FRONTEND_RESOURCES_PATH}/{self.path_cover_s}?ts={self.updated_at}" + if self.path_cover_s + else "" + ) + + @property + def path_cover_large(self) -> str: + return ( + f"{FRONTEND_RESOURCES_PATH}/{self.path_cover_l}?ts={self.updated_at}" + if self.path_cover_l + else "" + ) + + @property + def is_favorite(self) -> bool: + return self.name.lower() == "favourites" + def __repr__(self) -> str: return self.name diff --git a/backend/models/rom.py b/backend/models/rom.py index b3fc3a554..5ee9b2957 100644 --- a/backend/models/rom.py +++ b/backend/models/rom.py @@ -154,10 +154,6 @@ class Rom(BaseModel): def full_path(self) -> str: return f"{self.fs_path}/{self.fs_name}" - @cached_property - def has_cover(self) -> bool: - return bool(self.path_cover_s or self.path_cover_l) - @cached_property def merged_screenshots(self) -> list[str]: screenshots = [s.download_path for s in self.screenshots] @@ -175,6 +171,30 @@ class Rom(BaseModel): def fs_size_bytes(self) -> int: return sum(f.file_size_bytes for f in self.files) + @property + def fs_resources_path(self) -> str: + return f"roms/{str(self.platform_id)}/{str(self.id)}" + + @property + def path_cover_small(self) -> str: + return ( + f"{FRONTEND_RESOURCES_PATH}/{self.path_cover_s}?ts={self.updated_at}" + if self.path_cover_s + else "" + ) + + @property + def path_cover_large(self) -> str: + return ( + f"{FRONTEND_RESOURCES_PATH}/{self.path_cover_l}?ts={self.updated_at}" + if self.path_cover_l + else "" + ) + + @property + def is_unidentified(self) -> bool: + return not self.igdb_id and not self.moby_id + def get_collections(self) -> Sequence[Collection]: from handler.database import db_collection_handler @@ -268,10 +288,6 @@ class Rom(BaseModel): return [r["rating"] for r in self.igdb_metadata.get("age_ratings", [])] return [] - @property - def fs_resources_path(self) -> str: - return f"roms/{str(self.platform_id)}/{str(self.id)}" - def __repr__(self) -> str: return self.fs_name diff --git a/frontend/src/__generated__/models/CollectionSchema.ts b/frontend/src/__generated__/models/CollectionSchema.ts index 63ced7993..712068434 100644 --- a/frontend/src/__generated__/models/CollectionSchema.ts +++ b/frontend/src/__generated__/models/CollectionSchema.ts @@ -6,15 +6,15 @@ export type CollectionSchema = { id: number; name: string; description: string; - path_cover_l: (string | null); - path_cover_s: (string | null); - has_cover: boolean; + path_cover_small: (string | null); + path_cover_large: (string | null); url_cover: string; roms: Array; rom_count: number; user_id: number; user__username: string; is_public: boolean; + is_favorite: boolean; created_at: string; updated_at: string; }; diff --git a/frontend/src/__generated__/models/DetailedRomSchema.ts b/frontend/src/__generated__/models/DetailedRomSchema.ts index aa1a3203d..8ad9f5a28 100644 --- a/frontend/src/__generated__/models/DetailedRomSchema.ts +++ b/frontend/src/__generated__/models/DetailedRomSchema.ts @@ -45,10 +45,10 @@ export type DetailedRomSchema = { age_ratings: Array; igdb_metadata: (RomIGDBMetadata | null); moby_metadata: (RomMobyMetadata | null); - path_cover_s: (string | null); - path_cover_l: (string | null); - has_cover: boolean; + path_cover_small: (string | null); + path_cover_large: (string | null); url_cover: (string | null); + is_unidentified: boolean; revision: (string | null); regions: Array; languages: Array; diff --git a/frontend/src/__generated__/models/RomSchema.ts b/frontend/src/__generated__/models/RomSchema.ts index f0c9aad21..8f2674bd6 100644 --- a/frontend/src/__generated__/models/RomSchema.ts +++ b/frontend/src/__generated__/models/RomSchema.ts @@ -37,10 +37,10 @@ export type RomSchema = { age_ratings: Array; igdb_metadata: (RomIGDBMetadata | null); moby_metadata: (RomMobyMetadata | null); - path_cover_s: (string | null); - path_cover_l: (string | null); - has_cover: boolean; + path_cover_small: (string | null); + path_cover_large: (string | null); url_cover: (string | null); + is_unidentified: boolean; revision: (string | null); regions: Array; languages: Array; diff --git a/frontend/src/__generated__/models/SimpleRomSchema.ts b/frontend/src/__generated__/models/SimpleRomSchema.ts index fb9e98d37..5936d50a1 100644 --- a/frontend/src/__generated__/models/SimpleRomSchema.ts +++ b/frontend/src/__generated__/models/SimpleRomSchema.ts @@ -39,10 +39,10 @@ export type SimpleRomSchema = { age_ratings: Array; igdb_metadata: (RomIGDBMetadata | null); moby_metadata: (RomMobyMetadata | null); - path_cover_s: (string | null); - path_cover_l: (string | null); - has_cover: boolean; + path_cover_small: (string | null); + path_cover_large: (string | null); url_cover: (string | null); + is_unidentified: boolean; revision: (string | null); regions: Array; languages: Array; diff --git a/frontend/src/__generated__/models/VirtualCollectionSchema.ts b/frontend/src/__generated__/models/VirtualCollectionSchema.ts index cff8ab644..d8a67ef70 100644 --- a/frontend/src/__generated__/models/VirtualCollectionSchema.ts +++ b/frontend/src/__generated__/models/VirtualCollectionSchema.ts @@ -4,15 +4,10 @@ /* eslint-disable */ export type VirtualCollectionSchema = { id: string; - description: string; name: string; type: string; + description: string; roms: Array; - user_id: number; - is_public: boolean; rom_count: number; - has_cover: boolean; - created_at: string; - updated_at: string; }; diff --git a/frontend/src/components/Details/BackgroundHeader.vue b/frontend/src/components/Details/BackgroundHeader.vue index e46c21bb5..18d81dd78 100644 --- a/frontend/src/components/Details/BackgroundHeader.vue +++ b/frontend/src/components/Details/BackgroundHeader.vue @@ -14,9 +14,8 @@ const { currentRom } = storeToRefs(romsStore); @@ -78,7 +70,7 @@ const galleryViewStore = storeGalleryView();