refactor assets scan

backend/endpoints/sockets/scan.py

@@ -7,7 +7,18 @@ from exceptions.fs_exceptions import (
     FolderStructureNotMatchException,
     RomsNotFoundException,
 )
-from handler import fsasseth, dbh, fsplatformh, fsresourceh, fsromh, socketh
+from handler import (
+    dbh,
+    dbplatformh,
+    dbromh,
+    dbsaveh,
+    dbstateh,
+    fsasseth,
+    fsplatformh,
+    fsresourceh,
+    fsromh,
+    socketh,
+)
 from handler.redis_handler import high_prio_queue, redis_url
 from handler.scan_handler import (
     scan_platform,
@@ -17,6 +28,16 @@ from handler.scan_handler import (
     scan_state,
 )
 from logger.logger import log
+from handler.fs_handler import Asset
+
+
+def _get_socket_manager():
+    # Connect to external socketio server
+    return (
+        socketio.AsyncRedisManager(redis_url, write_only=True)
+        if ENABLE_EXPERIMENTAL_REDIS
+        else socketh.socket_server
+    )
 
 
 async def scan_platforms(
@@ -34,12 +55,7 @@ async def scan_platforms(
         selected_roms (list[str], optional): List of selected roms to be scanned. Defaults to ().
     """
 
-    # Connect to external socketio server
-    sm = (
-        socketio.AsyncRedisManager(redis_url, write_only=True)
-        if ENABLE_EXPERIMENTAL_REDIS
-        else socketh.socket_server
-    )
+    sm = _get_socket_manager()
 
     # Scanning file system
     try:
@@ -61,8 +77,7 @@ async def scan_platforms(
 
     for platform_slug in platform_list:
         scanned_platform = scan_platform(platform_slug, fs_platforms)
-        _added_platform = dbh.add_platform(scanned_platform)
-        platform = dbh.get_platform(_added_platform.id)
+        platform = dbplatformh.add_platform(scanned_platform)
 
         await sm.emit(
             "scan:scanning_platform",
@@ -71,7 +86,7 @@ async def scan_platforms(
 
         # Scanning roms
         try:
-            fs_roms = fsromh.get_roms(platform.fs_slug)
+            fs_roms = fsromh.get_roms(platform)
         except RomsNotFoundException as e:
             log.error(e)
             continue
@@ -84,7 +99,7 @@ async def scan_platforms(
             log.info(f"  {len(fs_roms)} roms found")
 
         for fs_rom in fs_roms:
-            rom = dbh.get_rom_by_filename(platform.id, fs_rom["file_name"])
+            rom = dbromh.get_rom_by_filename(platform.id, fs_rom["file_name"])
             if (rom and rom.id not in selected_roms and not complete_rescan) and not (
                 rescan_unidentified and rom and not rom.igdb_id
             ):
@@ -95,8 +110,8 @@ async def scan_platforms(
                 scanned_rom.id = rom.id
 
             scanned_rom.platform_id = platform.id
-            _added_rom = dbh.add_rom(scanned_rom)
-            rom = dbh.get_rom(_added_rom.id)
+            _added_rom = dbromh.add_rom(scanned_rom)
+            rom = dbromh.get_roms(_added_rom.id)
 
             await sm.emit(
                 "scan:scanning_rom",
@@ -106,116 +121,116 @@ async def scan_platforms(
                 },
             )
 
-        fs_assets = fsasseth.get_assets(platform.fs_slug)
+            fs_saves = fsasseth.get_assets(platform.fs_slug, rom.file_name_no_tags, Asset.SAVES)
 
-        # Scanning saves
-        log.info(f"\t · {len(fs_assets['saves'])} saves found")
-        for fs_emulator, fs_save_filename in fs_assets["saves"]:
-            scanned_save = scan_save(
-                platform=platform,
-                file_name=fs_save_filename,
-                emulator=fs_emulator,
-            )
+            # Scanning saves
+            log.info(f"\t · {len(fs_saves)} saves found")
+            for fs_emulator, fs_save_filename in fs_saves:
+                scanned_save = scan_save(
+                    platform=platform,
+                    file_name=fs_save_filename,
+                    emulator=fs_emulator,
+                )
 
-            save = dbh.get_save_by_filename(platform.id, fs_save_filename)
-            if save:
-                # Update file size if changed
-                if save.file_size_bytes != scanned_save.file_size_bytes:
-                    dbh.update_save(
-                        save.id, {"file_size_bytes": scanned_save.file_size_bytes}
-                    )
-                continue
+                save = dbsaveh.get_save_by_filename(rom.id, fs_save_filename)
+                if save:
+                    # Update file size if changed
+                    if save.file_size_bytes != scanned_save.file_size_bytes:
+                        dbh.update_save(
+                            save.id, {"file_size_bytes": scanned_save.file_size_bytes}
+                        )
+                    continue
 
-            scanned_save.emulator = fs_emulator
+                scanned_save.emulator = fs_emulator
 
-            rom = dbh.get_rom_by_filename_no_tags(scanned_save.file_name_no_tags)
-            if rom:
-                scanned_save.rom_id = rom.id
-                dbh.add_save(scanned_save)
+                rom = dbromh.get_rom_by_filename_no_tags(scanned_save.file_name_no_tags)
+                if rom:
+                    scanned_save.rom_id = rom.id
+                    dbsaveh.add_save(scanned_save)
 
-        # Scanning states
-        log.info(f"\t · {len(fs_assets['states'])} states found")
-        for fs_emulator, fs_state_filename in fs_assets["states"]:
-            scanned_state = scan_state(
-                platform=platform,
-                emulator=fs_emulator,
-                file_name=fs_state_filename,
-            )
+        # # Scanning states
+        # log.info(f"\t · {len(fs_assets['states'])} states found")
+        # for fs_emulator, fs_state_filename in fs_assets["states"]:
+        #     scanned_state = scan_state(
+        #         platform=platform,
+        #         emulator=fs_emulator,
+        #         file_name=fs_state_filename,
+        #     )
 
-            state = dbh.get_state_by_filename(platform.id, fs_state_filename)
-            if state:
-                # Update file size if changed
-                if state.file_size_bytes != scanned_state.file_size_bytes:
-                    dbh.update_state(
-                        state.id, {"file_size_bytes": scanned_state.file_size_bytes}
-                    )
+        #     state = dbstateh.get_state_by_filename(rom.id, fs_state_filename)
+        #     if state:
+        #         # Update file size if changed
+        #         if state.file_size_bytes != scanned_state.file_size_bytes:
+        #             dbh.update_state(
+        #                 state.id, {"file_size_bytes": scanned_state.file_size_bytes}
+        #             )
 
-                continue
+        #         continue
 
-            scanned_state.emulator = fs_emulator
-            # scanned_state.platform_slug = scanned_platform.slug TODO: remove
+        #     scanned_state.emulator = fs_emulator
 
-            rom = dbh.get_rom_by_filename_no_tags(scanned_state.file_name_no_tags)
-            if rom:
-                scanned_state.rom_id = rom.id
-                dbh.add_state(scanned_state)
+        #     rom = dbh.get_rom_by_filename_no_tags(scanned_state.file_name_no_tags)
+        #     if rom:
+        #         scanned_state.rom_id = rom.id
+        #         dbh.add_state(scanned_state)
 
-        # Scanning screenshots
-        log.info(f"\t · {len(fs_assets['screenshots'])} screenshots found")
-        for fs_screenshot_filename in fs_assets["screenshots"]:
-            scanned_screenshot = scan_screenshot(
-                file_name=fs_screenshot_filename, platform=platform
-            )
+        # # Scanning screenshots
+        # log.info(f"\t · {len(fs_assets['screenshots'])} screenshots found")
+        # for fs_screenshot_filename in fs_assets["screenshots"]:
+        #     scanned_screenshot = scan_screenshot(
+        #         file_name=fs_screenshot_filename, platform=platform
+        #     )
 
-            screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename)
-            if screenshot:
-                # Update file size if changed
-                if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes:
-                    dbh.update_screenshot(
-                        screenshot.id,
-                        {"file_size_bytes": scanned_screenshot.file_size_bytes},
-                    )
-                continue
+        #     screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename)
+        #     if screenshot:
+        #         # Update file size if changed
+        #         if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes:
+        #             dbh.update_screenshot(
+        #                 screenshot.id,
+        #                 {"file_size_bytes": scanned_screenshot.file_size_bytes},
+        #             )
+        #         continue
 
-            # scanned_screenshot.platform_slug = scanned_patform.slug TODO: remove
+        #     rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags)
 
-            rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags)
-            if rom:
-                scanned_screenshot.rom_id = rom.id
-                dbh.add_screenshot(scanned_screenshot)
+        #     if rom:
+        #         scanned_screenshot.rom_id = rom.id
+        #         dbh.add_screenshot(scanned_screenshot)
 
-        dbh.purge_saves(platform.id, [s for _e, s in fs_assets["saves"]])
-        dbh.purge_states(platform.id, [s for _e, s in fs_assets["states"]])
-        dbh.purge_screenshots(platform.id, fs_assets["screenshots"])
-        dbh.purge_roms(platform.id, [rom["file_name"] for rom in fs_roms])
+        # for fs_rom in fs_roms:
+        #     rom = dbh.get_rom_by_filename(platform.id, fs_rom["file_name"])
+        #     dbsaveh.purge_saves(rom.id, [s for _e, s in fs_assets["saves"]])
+        #     dbstateh.purge_states(rom.id, [s for _e, s in fs_assets["states"]])
+        #     dbh.purge_screenshots(rom.id, fs_assets["screenshots"])
+        # dbromh.purge_roms(platform.id, [rom["file_name"] for rom in fs_roms])
 
     # Scanning screenshots outside platform folders
-    fs_screenshots = fsasseth.get_screenshots()
-    log.info("Screenshots")
-    log.info(f" · {len(fs_screenshots)} screenshots found")
-    for fs_platform, fs_screenshot_filename in fs_screenshots:
-        scanned_screenshot = scan_screenshot(
-            file_name=fs_screenshot_filename, fs_platform=fs_platform
-        )
+    # fs_screenshots = fsasseth.get_screenshots()
+    # log.info("Screenshots")
+    # log.info(f" · {len(fs_screenshots)} screenshots found")
+    # for fs_platform, fs_screenshot_filename in fs_screenshots:
+    #     scanned_screenshot = scan_screenshot(
+    #         file_name=fs_screenshot_filename, fs_platform=fs_platform
+    #     )
 
-        screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename)
-        if screenshot:
-            # Update file size if changed
-            if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes:
-                dbh.update_screenshot(
-                    screenshot.id,
-                    {"file_size_bytes": scanned_screenshot.file_size_bytes},
-                )
-            continue
+    #     screenshot = dbh.get_screenshot_by_filename(fs_screenshot_filename)
+    #     if screenshot:
+    #         # Update file size if changed
+    #         if screenshot.file_size_bytes != scanned_screenshot.file_size_bytes:
+    #             dbh.update_screenshot(
+    #                 screenshot.id,
+    #                 {"file_size_bytes": scanned_screenshot.file_size_bytes},
+    #             )
+    #         continue
 
-        rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags)
-        if rom:
-            scanned_screenshot.rom_id = rom.id
-            # scanned_screenshot.platform_slug = rom.platform_slug TODO: remove
-            dbh.add_screenshot(scanned_screenshot)
+    #     rom = dbh.get_rom_by_filename_no_tags(scanned_screenshot.file_name_no_tags)
+    #     if rom:
+    #         scanned_screenshot.rom_id = rom.id
+    #         # scanned_screenshot.platform_slug = rom.platform_slug TODO: remove
+    #         dbh.add_screenshot(scanned_screenshot)
 
-    dbh.purge_screenshots([s for _e, s in fs_screenshots])
-    dbh.purge_platforms(fs_platforms)
+    # dbh.purge_screenshots([s for _e, s in fs_screenshots])
+    dbplatformh.purge_platforms(fs_platforms)
 
     log.info(emoji.emojize(":check_mark:  Scan completed "))
 

backend/handler/db_handler/db_handler.py

@@ -10,27 +10,6 @@ class DBHandler:
         self.engine = create_engine(ConfigManager.get_db_engine(), pool_pre_ping=True)
         self.session = sessionmaker(bind=self.engine, expire_on_commit=False)
 
-    # ==== Utils ======
-    @begin_session
-    def get_rom_by_filename(
-        self, platform_id: int, file_name: str, session: Session = None
-    ):
-        return session.scalars(
-            select(Rom).filter_by(platform_id=platform_id, file_name=file_name).limit(1)
-        ).first()
-
-    @begin_session
-    def get_rom_by_filename_no_tags(
-        self, file_name_no_tags: str, session: Session = None
-    ):
-        return session.scalars(
-            select(Rom).filter_by(file_name_no_tags=file_name_no_tags).limit(1)
-        ).first()
-
-    # ========= Saves =========
-
-    # ========= States =========
-
     # ========= Screenshots =========
     @begin_session
     def add_screenshot(self, screenshot: Screenshot, session: Session = None):
@@ -65,12 +44,12 @@ class DBHandler:
 
     @begin_session
     def purge_screenshots(
-        self, platform_id: int, screenshots: list[str], session: Session = None
+        self, rom_id: int, screenshots: list[str], session: Session = None
     ):
         return session.execute(
             delete(Screenshot)
             .where(
-                Screenshot.platform_id == platform_id,
+                Screenshot.rom_id == rom_id,
                 Screenshot.file_name.not_in(screenshots),
             )
             .execution_options(synchronize_session="evaluate")

backend/handler/db_handler/db_platforms_handler.py

@@ -45,23 +45,13 @@ class DBPlatformsHandler(DBHandler):
 
     @begin_session
     def purge_platforms(self, platforms: list[str], session: Session = None):
-        session.execute(
-            delete(Save)
-            .where(Save.platform_slug.not_in(platforms))
-            .execution_options(synchronize_session="evaluate")
-        )
-        session.execute(
-            delete(State)
-            .where(State.platform_slug.not_in(platforms))
-            .execution_options(synchronize_session="evaluate")
-        )
         return session.execute(
             delete(Platform)
             .where(or_(Platform.fs_slug.not_in(platforms), Platform.slug.is_(None)))
             .where(
                 select(func.count())
                 .select_from(Rom)
-                .filter_by(platform_slug=Platform.slug)
+                .filter_by(platform_id=Platform.id)
                 .as_scalar()
                 == 0
             )

backend/handler/db_handler/db_roms_handler.py

@@ -30,6 +30,10 @@ class DBRomsHandler(DBHandler):
         else:
             return data.order_by(_column.asc())
 
+    @begin_session
+    def add_rom(self, rom: Rom, session: Session = None):
+        return session.merge(rom)
+
     @begin_session
     def get_roms(
         self,
@@ -50,6 +54,22 @@ class DBRomsHandler(DBHandler):
             )
         )
 
+    @begin_session
+    def get_rom_by_filename(
+        self, platform_id: int, file_name: str, session: Session = None
+    ):
+        return session.scalars(
+            select(Rom).filter_by(platform_id=platform_id, file_name=file_name).limit(1)
+        ).first()
+
+    @begin_session
+    def get_rom_by_filename_no_tags(
+        self, file_name_no_tags: str, session: Session = None
+    ):
+        return session.scalars(
+            select(Rom).filter_by(file_name_no_tags=file_name_no_tags).limit(1)
+        ).first()
+
     @begin_session
     def update_rom(self, id: int, data: dict, session: Session = None):
         return session.execute(

backend/handler/db_handler/db_saves_handler.py

@@ -1,7 +1,7 @@
 from decorators.database import begin_session
 from handler.db_handler.db_handler import DBHandler
 from models import Save
-from sqlalchemy import and_, delete, update
+from sqlalchemy import and_, delete, select, update
 from sqlalchemy.orm import Session
 
 
@@ -14,6 +14,14 @@ class DBSavesHandler(DBHandler):
     def get_save(self, id: int, session: Session = None):
         return session.get(Save, id)
 
+    @begin_session
+    def get_save_by_filename(
+        self, rom_id: str, file_name: str, session: Session = None
+    ):
+        return session.scalars(
+            select(Save).filter_by(rom_id=rom_id, file_name=file_name).limit(1)
+        ).first()
+
     @begin_session
     def update_save(self, id: int, data: dict, session: Session = None):
         session.execute(
@@ -32,9 +40,9 @@ class DBSavesHandler(DBHandler):
         )
 
     @begin_session
-    def purge_saves(self, platform_id: int, saves: list[str], session: Session = None):
+    def purge_saves(self, rom_id: int, saves: list[str], session: Session = None):
         return session.execute(
             delete(Save)
-            .where(and_(Save.platform_id == platform_id, Save.file_name.not_in(saves)))
+            .where(and_(Save.rom_id == rom_id, Save.file_name.not_in(saves)))
             .execution_options(synchronize_session="evaluate")
         )

backend/handler/db_handler/db_states_handler.py

@@ -1,7 +1,7 @@
 from decorators.database import begin_session
 from handler.db_handler.db_handler import DBHandler
 from models import State
-from sqlalchemy import and_, delete, update
+from sqlalchemy import and_, delete, select, update
 from sqlalchemy.orm import Session
 
 
@@ -14,6 +14,14 @@ class DBStatesHandler(DBHandler):
     def get_state(self, id: int, session: Session = None):
         return session.get(State, id)
 
+    @begin_session
+    def get_state_by_filename(
+        self, rom_id: str, file_name: str, session: Session = None
+    ):
+        return session.scalars(
+            select(State).filter_by(rom_id=rom_id, file_name=file_name).limit(1)
+        ).first()
+
     @begin_session
     def update_state(self, id: int, data: dict, session: Session = None):
         session.execute(
@@ -32,13 +40,9 @@ class DBStatesHandler(DBHandler):
         )
 
     @begin_session
-    def purge_states(
-        self, platform_id: int, states: list[str], session: Session = None
-    ):
+    def purge_states(self, rom_id: int, states: list[str], session: Session = None):
         return session.execute(
             delete(State)
-            .where(
-                and_(State.platform_id == platform_id, State.file_name.not_in(states))
-            )
+            .where(and_(State.rom_id == rom_id, State.file_name.not_in(states)))
             .execution_options(synchronize_session="evaluate")
         )

backend/handler/fs_handler/__init__.py

@@ -73,3 +73,9 @@ EXTENSION_REGEX = r"\.(([a-z]+\.)*\w+)$"
 class CoverSize(Enum):
     SMALL = "small"
     BIG = "big"
+
+
+class Asset(Enum):
+    SAVES = "saves"
+    STATES = "states"
+    SCREENSHOTS = "screenshots"

backend/handler/fs_handler/fs_assets_handler.py

@@ -7,7 +7,7 @@ import requests
 from config import LIBRARY_BASE_PATH
 from config.config_manager import config_manager as cm
 from fastapi import UploadFile
-from handler.fs_handler import RESOURCES_BASE_PATH
+from handler.fs_handler import RESOURCES_BASE_PATH, Asset
 from handler.fs_handler.fs_handler import FSHandler
 from logger.logger import log
 
@@ -71,64 +71,75 @@ class FSAssetsHandler(FSHandler):
 
         return {"path_screenshots": path_screenshots}
 
-    def get_assets(self, platform_slug: str):
-        saves_path = self.get_fs_structure(
-            platform_slug, folder=cm.config.SAVES_FOLDER_NAME
-        )
-        saves_file_path = f"{LIBRARY_BASE_PATH}/{saves_path}"
+    def get_assets(
+        self, platform_slug: str, rom_file_name_no_tags: str, asset_type: Asset
+    ):
+        asset_folder_name = {
+            Asset.SAVES: cm.config.SAVES_FOLDER_NAME,
+            Asset.STATES: cm.config.STATES_FOLDER_NAME,
+            Asset.SCREENSHOTS: cm.config.SCREENSHOTS_FOLDER_NAME,
+        }
 
-        fs_saves: list[str] = []
-        fs_states: list[str] = []
-        fs_screenshots: list[str] = []
+        assets_path = self.get_fs_structure(
+            platform_slug, folder=asset_folder_name[asset_type]
+        )
+
+        saves_file_path = f"{LIBRARY_BASE_PATH}/{assets_path}"
+
+        fs_assets: list[str] = []
+        # fs_states: list[str] = []
+        # fs_screenshots: list[str] = []
 
         try:
             emulators = list(os.walk(saves_file_path))[0][1]
             for emulator in emulators:
-                fs_saves += [
+                fs_assets += [
                     (emulator, file)
                     for file in list(os.walk(f"{saves_file_path}/{emulator}"))[0][2]
                 ]
 
-            fs_saves += [(None, file) for file in list(os.walk(saves_file_path))[0][2]]
-        except IndexError:
-            pass
-
-        states_path = self.get_fs_structure(
-            platform_slug, folder=cm.config.STATES_FOLDER_NAME
-        )
-        states_file_path = f"{LIBRARY_BASE_PATH}/{states_path}"
-
-        try:
-            emulators = list(os.walk(states_file_path))[0][1]
-            for emulator in emulators:
-                fs_states += [
-                    (emulator, file)
-                    for file in list(os.walk(f"{states_file_path}/{emulator}"))[0][2]
-                ]
-
-            fs_states += [
-                (None, file) for file in list(os.walk(states_file_path))[0][2]
+            fs_assets += [
+                ("", file)
+                for file in list(os.walk(saves_file_path))[0][2]
+                if file.split(".")[0] == rom_file_name_no_tags
             ]
         except IndexError:
             pass
 
-        screenshots_path = self.get_fs_structure(
-            platform_slug, folder=cm.config.SCREENSHOTS_FOLDER_NAME
-        )
-        screenshots_file_path = f"{LIBRARY_BASE_PATH}/{screenshots_path}"
+        # states_path = self.get_fs_structure(
+        #     platform_slug, folder=cm.config.STATES_FOLDER_NAME
+        # )
+        # states_file_path = f"{LIBRARY_BASE_PATH}/{states_path}"
 
-        try:
-            fs_screenshots += [
-                file for file in list(os.walk(screenshots_file_path))[0][2]
-            ]
-        except IndexError:
-            pass
+        # try:
+        #     emulators = list(os.walk(states_file_path))[0][1]
+        #     for emulator in emulators:
+        #         fs_states += [
+        #             (emulator, file)
+        #             for file in list(os.walk(f"{states_file_path}/{emulator}"))[0][2]
+        #         ]
 
-        return {
-            "saves": fs_saves,
-            "states": fs_states,
-            "screenshots": fs_screenshots,
-        }
+        #     fs_states += [
+        #         (None, file) for file in list(os.walk(states_file_path))[0][2]
+        #     ]
+        # except IndexError:
+        #     pass
+
+        # screenshots_path = self.get_fs_structure(
+        #     platform_slug, folder=cm.config.SCREENSHOTS_FOLDER_NAME
+        # )
+        # screenshots_file_path = f"{LIBRARY_BASE_PATH}/{screenshots_path}"
+
+        # try:
+        #     fs_screenshots += [
+        #         file for file in list(os.walk(screenshots_file_path))[0][2]
+        #     ]
+        # except IndexError:
+        #     pass
+
+        return fs_assets
+        # "states": fs_states,
+        # "screenshots": fs_screenshots,
 
     @staticmethod
     def get_screenshots():