From 5b494c3bc4acdb72fdc2652da4e49459fdff0467 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 20:46:50 +0000 Subject: [PATCH 1/4] feat: opt-in per-platform subfolder scanning with non-destructive identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds opt-in, per-platform subfolder scanning (closes #2050). When enabled for a platform, RomM recurses that platform's subfolders and treats each nested file as its own ROM, instead of collapsing a subfolder into one multi-file ROM — so libraries organized into Hacks/, Translations/, Homebrew/, etc. are scanned correctly. Enable per platform in config.yml (default off, existing setups unaffected): scan: subfolders: nes: true # recurse every subfolder snes: ["Hacks", "Translations"] # recurse only these; keep others whole Key difference from a naive path-identity approach: identity is reconciled by content hash on scan. Moving or renaming a ROM between subfolders is detected by its hash against a now-missing ROM and relocated in place, so saves, play history, favorites and collection membership follow it. Falls back to path identity when hashing is unavailable (skip_hash_calculation or a non-hashable platform). A cheap size pre-filter avoids hashing every newly-seen file (e.g. on first enable). Details: - Per-platform opt-in via scan.subfolders (fs_slug -> bool | list[str]); no DB schema change. - Recursion skips hidden (dot-prefixed) folders, and keeps a folder whole as a single multi-file ROM when it holds a disc/playlist descriptor (.m3u/.cue/.gdi/.ccd/.toc) — covers cue+bin / multi-disc games. - Path-based keying for the steady-state lookup (get_roms_by_fs_name and mark_missing_roms key on full path) so identically-named files in different subfolders stay distinct; the (platform_id, fs_name) index is non-unique. - File resolution (download/delete/hash) uses the ROM's stored fs_path. - Scan log flags entries that became "missing" because a folder is now recursed, so stale entries are easy to clean up. - Frontend: the Files tab shows a ROM's on-disk Location (click-to-copy), built as a shared LocationChip mirroring HashChip's RTag-based pattern. New i18n keys added to all locales. Tests: backend unit tests for recursion (collisions, hidden folders, descriptor dirs kept whole, named-list form), full-path keying in get_roms_by_fs_name / mark_missing_roms, and hash-based relocation. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Sk2dzM3K9qxWBdPAeGb7us --- backend/config/config_manager.py | 37 +++ backend/endpoints/sockets/scan.py | 205 ++++++++++++++- backend/handler/database/roms_handler.py | 58 ++++- backend/handler/filesystem/roms_handler.py | 152 ++++++++--- backend/tests/endpoints/sockets/test_scan.py | 107 +++++++- .../handler/filesystem/test_roms_handler.py | 238 ++++++++++++++++-- backend/tests/handler/test_db_handler.py | 53 +++- backend/tests/handler/test_fastapi.py | 4 + examples/config.example.yml | 32 +++ frontend/src/locales/bg_BG/rom.json | 4 +- frontend/src/locales/cs_CZ/rom.json | 4 +- frontend/src/locales/de_DE/rom.json | 4 +- frontend/src/locales/en_GB/rom.json | 4 +- frontend/src/locales/en_US/rom.json | 4 +- frontend/src/locales/es_ES/rom.json | 4 +- frontend/src/locales/fr_FR/rom.json | 4 +- frontend/src/locales/hu_HU/rom.json | 4 +- frontend/src/locales/it_IT/rom.json | 4 +- frontend/src/locales/ja_JP/rom.json | 4 +- frontend/src/locales/ko_KR/rom.json | 4 +- frontend/src/locales/pl_PL/rom.json | 4 +- frontend/src/locales/pt_BR/rom.json | 4 +- frontend/src/locales/ro_RO/rom.json | 4 +- frontend/src/locales/ru_RU/rom.json | 4 +- frontend/src/locales/zh_CN/rom.json | 4 +- frontend/src/locales/zh_TW/rom.json | 4 +- .../GameDetails/FilesTab/FilesSummary.vue | 15 ++ .../src/v2/components/shared/LocationChip.vue | 82 ++++++ 28 files changed, 951 insertions(+), 100 deletions(-) create mode 100644 frontend/src/v2/components/shared/LocationChip.vue diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index d1cfc842f0..91edc46267 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -132,6 +132,7 @@ class Config: SCAN_REGION_PRIORITY: list[str] SCAN_LANGUAGE_PRIORITY: list[str] SCAN_MEDIA: list[str] + SCAN_SUBFOLDERS: dict[str, bool | list[str]] GAMELIST_MEDIA_THUMBNAIL: MetadataMediaType GAMELIST_MEDIA_IMAGE: MetadataMediaType @@ -160,6 +161,25 @@ def has_structure_path_b(self) -> bool: return False + def subfolder_scan_spec(self, fs_slug: str) -> bool | frozenset[str]: + """How the scanner should recurse into a platform's subfolders. + + Opt-in per platform via `scan.subfolders` in config.yml, where the + value is either: + - ``True`` -> recurse every subfolder (each nested file becomes its + own rom); + - a list of folder names -> recurse only those folders, leaving every + other folder as a single multi-file rom (so folder-based multi-file + games elsewhere on the platform stay intact); + - ``False`` / omitted -> don't recurse (default behavior). + + Returns ``True``, a ``frozenset`` of folder names, or ``False``. + """ + value = getattr(self, "SCAN_SUBFOLDERS", {}).get(fs_slug, False) + if isinstance(value, list): + return frozenset(value) + return bool(value) + class ConfigManager: """ @@ -446,6 +466,7 @@ def _parse_config(self): PEGASUS_AUTO_EXPORT_ON_SCAN=pydash.get( self._raw_config, "scan.pegasus.export", False ), + SCAN_SUBFOLDERS=pydash.get(self._raw_config, "scan.subfolders", {}), ) def _get_ejs_controls(self) -> dict[str, EjsControls]: @@ -660,6 +681,21 @@ def _validate_config(self): log.critical("Invalid config.yml: scan.media must be a list") sys.exit(3) + if not isinstance(self.config.SCAN_SUBFOLDERS, dict): + log.critical("Invalid config.yml: scan.subfolders must be a dictionary") + sys.exit(3) + for fs_slug, value in self.config.SCAN_SUBFOLDERS.items(): + is_bool = isinstance(value, bool) + is_str_list = isinstance(value, list) and all( + isinstance(name, str) for name in value + ) + if not (is_bool or is_str_list): + log.critical( + f"Invalid config.yml: scan.subfolders.{fs_slug} must be a " + "boolean or a list of folder names" + ) + sys.exit(3) + # Drop unknown media types rather than exiting, since a newer release # may ship sample configs referencing media types this version doesn't know. unknown_media = [ @@ -779,6 +815,7 @@ def _update_config_file(self) -> None: "language": self.config.SCAN_LANGUAGE_PRIORITY, }, "media": self.config.SCAN_MEDIA, + "subfolders": self.config.SCAN_SUBFOLDERS, "gamelist": { "export": self.config.GAMELIST_AUTO_EXPORT_ON_SCAN, "media": { diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index f9f8e92c3a..e12be729be 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import os +from collections import defaultdict from dataclasses import dataclass from itertools import batched from typing import Any, Final @@ -30,7 +32,7 @@ fs_resource_handler, fs_rom_handler, ) -from handler.filesystem.roms_handler import FSRom +from handler.filesystem.roms_handler import FSRom, ParsedRomFiles from handler.metadata import meta_gamelist_handler, meta_hltb_handler from handler.metadata.ss_handler import add_ss_auth_to_url, get_preferred_media_types from handler.redis_handler import get_job_func_name, high_prio_queue, redis_client @@ -247,7 +249,9 @@ async def _identify_rom( # Update properties that don't require metadata parsed_tags = fs_rom_handler.parse_tags(fs_rom["fs_name"]) - roms_path = fs_rom_handler.get_roms_fs_structure(platform.fs_slug) + # The discovered path is the rom's actual directory, which may be a + # subfolder of the platform roms folder when subfolder scanning is enabled. + roms_path = fs_rom["fs_path"] # Create the entry early so we have the ID newly_added: bool = rom is None @@ -491,6 +495,159 @@ async def _identify_rom( ) +def _fs_rom_size(fs_rom: FSRom) -> int: + """On-disk size of a discovered rom (a file's size, or the sum of a + folder's files). Cheap stat-only pre-filter used before hashing during + relocation matching. Returns -1 when the path can't be read. + """ + abs_path = os.path.join( + fs_rom_handler.base_path, fs_rom["fs_path"], fs_rom["fs_name"] + ) + try: + if os.path.isdir(abs_path): + total = 0 + for root, _dirs, files in os.walk(abs_path): + for name in files: + try: + total += os.stat(os.path.join(root, name)).st_size + except OSError: + pass + return total + return os.stat(abs_path).st_size + except OSError: + return -1 + + +def _hashes_match(rom: Rom, parsed: ParsedRomFiles) -> bool: + """Whether an existing rom and freshly parsed files are the same content. + + A single matching non-empty hash is enough — a collision across sha1/md5/ + crc/ra is negligible, and different platforms populate different hashes. + """ + for stored, computed in ( + (rom.sha1_hash, parsed.sha1_hash), + (rom.md5_hash, parsed.md5_hash), + (rom.crc_hash, parsed.crc_hash), + (rom.ra_hash, parsed.ra_hash), + ): + if stored and computed and stored == computed: + return True + return False + + +async def _reconcile_relocated_roms( + platform: Platform, + fs_roms: list[FSRom], +) -> set[str]: + """Relocate roms whose on-disk path changed instead of re-importing them. + + When subfolder scanning is enabled, moving (or renaming) a file reads as a + new path. Rather than insert a fresh rom and mark the old one missing — + which would drop saves, play history, favorites and collection membership — + match a newly-seen file to a now-missing rom by content hash and update that + rom's path in place. Returns the set of full paths that were fully handled + this way (so the caller skips them in the normal scan loop). + + Falls back to no-op (path-based identity) when hashes are unavailable + (``skip_hash_calculation`` or a non-hashable platform). + """ + existing = db_rom_handler.get_roms_for_relocation(platform.id) + existing_paths = {rom.full_path for rom in existing} + + present_paths = {f"{fr['fs_path']}/{fr['fs_name']}" for fr in fs_roms} + # A rom that vanished from its stored path is a relocation candidate. + disappeared = [rom for rom in existing if rom.full_path not in present_paths] + if not disappeared: + return set() + + # Group candidates by total size: a moved file keeps its size, so this is a + # cheap pre-filter that avoids hashing every newly-seen file (e.g. on first + # enable, where the whole subfolder tree reads as new). + by_size: dict[int, list[Rom]] = defaultdict(list) + for rom in disappeared: + by_size[rom.fs_size_bytes].append(rom) + + handled: set[str] = set() + claimed: set[int] = set() + calculate_hashes = not cm.get_config().SKIP_HASH_CALCULATION + + for fs_rom in fs_roms: + full_path = f"{fs_rom['fs_path']}/{fs_rom['fs_name']}" + if full_path in existing_paths: + continue # already matched by path; not a relocation + + candidates = [ + rom + for rom in by_size.get(_fs_rom_size(fs_rom), []) + if rom.id not in claimed + ] + if not candidates: + continue + + # Compute the file's identity hashes (and its rom files for the new + # location) once; reuse them for both matching and the relocation write. + transient = Rom( + fs_name=fs_rom["fs_name"], + fs_path=fs_rom["fs_path"], + platform_id=platform.id, + ) + transient.platform = platform + parsed = await fs_rom_handler.get_rom_files( + transient, calculate_hashes=calculate_hashes + ) + if not ( + parsed.sha1_hash or parsed.md5_hash or parsed.crc_hash or parsed.ra_hash + ): + continue # no usable hash to match on + + match = next((rom for rom in candidates if _hashes_match(rom, parsed)), None) + if match is None: + continue + + claimed.add(match.id) + handled.add(full_path) + + old_path = match.full_path + db_rom_handler.update_rom( + match.id, + { + "fs_name": fs_rom["fs_name"], + "fs_path": fs_rom["fs_path"], + "fs_size_bytes": sum(f.file_size_bytes for f in parsed.rom_files), + "crc_hash": parsed.crc_hash, + "md5_hash": parsed.md5_hash, + "sha1_hash": parsed.sha1_hash, + "ra_hash": parsed.ra_hash, + "missing_from_fs": False, + }, + ) + db_rom_handler.purge_rom_files(match.id) + for file in parsed.rom_files: + db_rom_handler.add_rom_file( + RomFile( + rom_id=match.id, + file_name=file.file_name, + file_path=file.file_path, + file_size_bytes=file.file_size_bytes, + last_modified=file.last_modified, + category=file.category, + audio_meta=file.audio_meta, + crc_hash=file.crc_hash, + md5_hash=file.md5_hash, + sha1_hash=file.sha1_hash, + ra_hash=file.ra_hash, + chd_sha1_hash=file.chd_sha1_hash, + ) + ) + + log.info( + f"{hl('Relocated', color=BLUE)} {hl(old_path)} → {hl(full_path)} " + "(moved on disk; metadata and user data preserved)" + ) + + return handled + + async def _identify_platform( platform_slug: str, scan_type: ScanType, @@ -582,6 +739,24 @@ async def _identify_platform( else: log.info(f"{hl(str(len(fs_roms)))} roms found in the file system") + # Detect roms that simply moved on disk (subfolder scanning) and relocate + # them in place so their saves/history/favorites/collections follow, + # instead of re-importing them as new and orphaning the old entry. Only + # runs when subfolder scanning is enabled for the platform, so default + # libraries pay no extra cost. + if cm.get_config().subfolder_scan_spec(platform.fs_slug): + relocated_paths = await _reconcile_relocated_roms(platform, fs_roms) + if relocated_paths: + await scan_stats.increment( + socket_manager=socket_manager, + scanned_roms=len(relocated_paths), + ) + fs_roms = [ + fs_rom + for fs_rom in fs_roms + if f"{fs_rom['fs_path']}/{fs_rom['fs_name']}" not in relocated_paths + ] + # Create semaphore to limit concurrent ROM scanning scan_semaphore = asyncio.Semaphore(SCAN_WORKERS) @@ -602,7 +777,10 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: ) for fs_roms_batch in batched(fs_roms, 200, strict=False): - roms_by_fs_name = db_rom_handler.get_roms_by_fs_name( + # Key matches on the rom's full path (fs_path/fs_name), not just the + # file name, so identically-named files in different subfolders don't + # collide when subfolder scanning is enabled. + roms_by_full_path = db_rom_handler.get_roms_by_fs_name( platform_id=platform.id, fs_names={fs_rom["fs_name"] for fs_rom in fs_roms_batch}, ) @@ -612,7 +790,7 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: roms_to_scan: list[tuple[FSRom, Rom | None]] = [] for fs_rom in fs_roms_batch: - rom = roms_by_fs_name.get(fs_rom["fs_name"]) + rom = roms_by_full_path.get(f"{fs_rom['fs_path']}/{fs_rom['fs_name']}") if _should_scan_rom( scan_type=scan_type, rom=rom, @@ -644,12 +822,27 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: log.error(f"Error scanning ROM {fs_rom['fs_name']}: {result}") missing_roms = db_rom_handler.mark_missing_roms( - platform.id, [rom["fs_name"] for rom in fs_roms] + platform.id, [f"{rom['fs_path']}/{rom['fs_name']}" for rom in fs_roms] ) if len(missing_roms) > 0: log.warning(f"{hl('Missing')} roms from filesystem:") + # A folder that subfolder scanning now recurses into used to be a single + # multi-file rom; that old entry shows up here as missing. Flag those so + # it's clear the "missing" is expected and the stale entry can be + # deleted. A superseded folder's path is a parent of a discovered rom. + recursed_paths = {rom["fs_path"] for rom in fs_roms} for r in missing_roms: - log.warning(f" - {r.fs_name}") + superseded = any( + p == r.full_path or p.startswith(f"{r.full_path}/") + for p in recursed_paths + ) + if superseded: + log.warning( + f" - {r.fs_name} (now scanned as a folder of roms — " + "delete this stale entry to clean up)" + ) + else: + log.warning(f" - {r.fs_name}") missing_firmware = db_firmware_handler.mark_missing_firmware( platform.id, [fw for fw in fs_firmware] diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index 479ebb9f35..34c4830eb5 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -1171,7 +1171,11 @@ def get_roms_by_fs_name( fs_names: Iterable[str], session: Session = None, # type: ignore ) -> dict[str, Rom]: - """Retrieve a dictionary of roms by their filesystem names. + """Retrieve a dictionary of roms keyed by their full path (fs_path/fs_name). + + Filters by file name for an indexed lookup, but keys the result on the + full path so identically-named files in different subfolders (subfolder + scanning) remain distinct. Eager-loads only `platform` (used downstream by the scan loop via `rom.platform_slug` / `rom.platform.fs_slug`). This deliberately @@ -1195,7 +1199,7 @@ def get_roms_by_fs_name( .all() ) - return {rom.fs_name: rom for rom in roms} + return {rom.full_path: rom for rom in roms} @begin_session def update_rom( @@ -1278,19 +1282,23 @@ def mark_missing_roms( ) -> Sequence[Rom]: """Sync `missing_from_fs` for a platform against the keep-list. + The keep-list holds rom full paths (fs_path/fs_name) so that + identically-named files in different subfolders are tracked + independently when subfolder scanning is enabled. + Reads the rows once and writes only those whose state actually changes, so a re-scan of an unchanged platform issues no updates. """ keep_set = set(fs_roms_to_keep) rows = session.execute( - select(Rom.id, Rom.fs_name, Rom.missing_from_fs).where( + select(Rom.id, Rom.fs_path, Rom.fs_name, Rom.missing_from_fs).where( Rom.platform_id == platform_id ) ).all() flips: dict[bool, list[int]] = {True: [], False: []} - for rom_id, fs_name, was_missing in rows: - is_missing = fs_name not in keep_set + for rom_id, fs_path, fs_name, was_missing in rows: + is_missing = f"{fs_path}/{fs_name}" not in keep_set if is_missing != was_missing: flips[is_missing].append(rom_id) @@ -1305,8 +1313,12 @@ def mark_missing_roms( return ( session.scalars( + # `fs_path` is eager-loaded alongside `fs_name` so callers can + # read `rom.full_path` after the session closes (the returned + # instances are detached); without it that lazy-loads and raises + # DetachedInstanceError. select(Rom) - .options(load_only(Rom.id, Rom.fs_name)) + .options(load_only(Rom.id, Rom.fs_name, Rom.fs_path)) .where( and_( Rom.platform_id == platform_id, @@ -1319,6 +1331,40 @@ def mark_missing_roms( .all() ) + @begin_session + def get_roms_for_relocation( + self, + platform_id: int, + session: Session = None, # type: ignore + ) -> Sequence[Rom]: + """Light-weight load of a platform's roms for relocation matching. + + Returns detached rom instances carrying only the columns needed to + detect a moved/renamed file by content (identity hashes + size) and to + resolve its current `full_path`. Used by the scan loop to relocate a + rom whose on-disk path changed instead of re-importing it as new. + """ + return ( + session.scalars( + select(Rom) + .options( + load_only( + Rom.id, + Rom.fs_name, + Rom.fs_path, + Rom.fs_size_bytes, + Rom.crc_hash, + Rom.md5_hash, + Rom.sha1_hash, + Rom.ra_hash, + ) + ) + .where(Rom.platform_id == platform_id) + ) + .unique() + .all() + ) + @begin_session def add_rom_user( self, diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index d1a319b52c..4d5de00a7b 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -88,6 +88,7 @@ class FSRom(TypedDict): fs_name: str + fs_path: str flat: bool nested: bool files: list[RomFile] @@ -316,10 +317,11 @@ async def get_rom_files( from adapters.services.rahasher import RAHasherService from handler.metadata import meta_ra_handler - rel_roms_path = self.get_roms_fs_structure( - rom.platform.fs_slug - ) # Relative path to roms - abs_fs_path = self.validate_path(rel_roms_path) # Absolute path to roms + # The rom's stored directory is the source of truth for its location, so + # roms inside a platform subfolder (subfolder scanning) resolve to their + # real path rather than the platform roms root. + rel_roms_path = rom.fs_path # Relative path to the rom's directory + abs_fs_path = self.validate_path(rel_roms_path) # Absolute path to that dir rom_files: list[RomFile] = [] # Skip hashing games for platforms that don't have a hash database or when hashes are disabled @@ -392,14 +394,19 @@ def _largest_chd_file() -> Path | None: try: if is_top_level: # Include this file in the main ROM hash calculation - crc_c, rom_crc_c, md5_h, rom_md5_h, sha1_h, rom_sha1_h = ( - await asyncio.to_thread( - self._calculate_rom_hashes, - Path(f_path, file_name), - rom_crc_c, - rom_md5_h, - rom_sha1_h, - ) + ( + crc_c, + rom_crc_c, + md5_h, + rom_md5_h, + sha1_h, + rom_sha1_h, + ) = await asyncio.to_thread( + self._calculate_rom_hashes, + Path(f_path, file_name), + rom_crc_c, + rom_md5_h, + rom_sha1_h, ) else: # Calculate individual file hash only @@ -529,14 +536,19 @@ def _hash_raw_archive(crc: int) -> int: ) elif hashable_platform: try: - crc_c, rom_crc_c, md5_h, rom_md5_h, sha1_h, rom_sha1_h = ( - await asyncio.to_thread( - self._calculate_rom_hashes, - Path(abs_fs_path, rom.fs_name), - rom_crc_c, - rom_md5_h, - rom_sha1_h, - ) + ( + crc_c, + rom_crc_c, + md5_h, + rom_md5_h, + sha1_h, + rom_sha1_h, + ) = await asyncio.to_thread( + self._calculate_rom_hashes, + Path(abs_fs_path, rom.fs_name), + rom_crc_c, + rom_md5_h, + rom_sha1_h, ) except zlib.error: crc_c = 0 @@ -664,21 +676,93 @@ def update_hashes(chunk: bytes | bytearray): rom_sha1_h, ) + # Disc/playlist descriptors that mark a folder as a single multi-file game + # (a multi-disc title), so it is kept whole instead of being recursed into + # when subfolder scanning is enabled — preventing it from being split into + # one rom per disc. + _MULTI_DISC_DESCRIPTOR_EXTS = (".m3u", ".cue", ".gdi", ".ccd", ".toc") + + @staticmethod + def _should_recurse_dir(directory: str, recurse: bool | frozenset[str]) -> bool: + """Whether to descend into ``directory`` (collecting its contents as + individual roms) rather than treat it as one multi-file rom. + + Hidden (dot-prefixed) folders are never descended into. Otherwise + ``recurse`` is ``True`` (all folders), a set of folder names (only + those), or ``False`` (none). + """ + if directory.startswith("."): + return False + if isinstance(recurse, frozenset): + return directory in recurse + return recurse + + async def _is_multi_disc_dir(self, rel_dir_path: str) -> bool: + """True if a directory directly contains a disc/playlist descriptor. + + Such a directory is a single multi-file rom (a multi-disc game), so it + is kept whole instead of being recursed into when subfolder scanning is + enabled — preventing a multi-disc game from being split into one rom per + disc. Covers ``.m3u`` playlists as well as ``.cue``/``.gdi``/``.ccd``/ + ``.toc`` track descriptors (the ``.cue``+``.bin`` case raised in review). + """ + return any( + f.lower().endswith(self._MULTI_DISC_DESCRIPTOR_EXTS) + for f in await self.list_files(rel_dir_path) + ) + + async def _collect_fs_roms( + self, rel_roms_path: str, recurse: bool | frozenset[str] + ) -> list[dict]: + """Discover roms under ``rel_roms_path``. + + Single files are flat roms (``fs_path`` = their parent directory). A + directory is a single multi-file rom unless ``recurse`` says to descend + into it, in which case its contents are collected as individual roms. + ``recurse`` is ``True`` (descend every folder), a set of folder names + (descend only those — every other folder stays a multi-file rom), or + ``False`` (descend none). + + Hidden (dot-prefixed) directories are never descended into, and a + directory holding a disc/playlist descriptor is kept whole as a single + multi-file rom (a multi-disc game) even while recursing. + """ + fs_roms: list[dict] = [ + {"fs_name": rom, "fs_path": rel_roms_path, "flat": True, "nested": False} + for rom in self.exclude_single_files(await self.list_files(rel_roms_path)) + ] + + for directory in self.exclude_multi_roms( + await self.list_directories(rel_roms_path) + ): + dir_path = f"{rel_roms_path}/{directory}" + if self._should_recurse_dir( + directory, recurse + ) and not await self._is_multi_disc_dir(dir_path): + fs_roms.extend(await self._collect_fs_roms(dir_path, recurse)) + else: + fs_roms.append( + { + "fs_name": directory, + "fs_path": rel_roms_path, + "flat": False, + "nested": True, + } + ) + + return fs_roms + async def count_roms(self, platform: Platform) -> int: """Return the number of filesystem roms for a platform without materializing FSRom objects. """ + recurse = cm.get_config().subfolder_scan_spec(platform.fs_slug) try: rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) - fs_single_roms = await self.list_files(path=rel_roms_path) - fs_multi_roms = await self.list_directories(path=rel_roms_path) + return len(await self._collect_fs_roms(rel_roms_path, recurse)) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e - return len(self.exclude_single_files(fs_single_roms)) + len( - self.exclude_multi_roms(fs_multi_roms) - ) - async def get_roms(self, platform: Platform) -> list[FSRom]: """Gets all filesystem roms for a platform @@ -687,28 +771,20 @@ async def get_roms(self, platform: Platform) -> list[FSRom]: Returns: list with all the filesystem roms for a platform """ + recurse = cm.get_config().subfolder_scan_spec(platform.fs_slug) try: rel_roms_path = self.get_roms_fs_structure( platform.fs_slug ) # Relative path to roms - - fs_single_roms = await self.list_files(path=rel_roms_path) - fs_multi_roms = await self.list_directories(path=rel_roms_path) + fs_roms = await self._collect_fs_roms(rel_roms_path, recurse) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e - fs_roms: list[dict] = [ - {"fs_name": rom, "flat": True, "nested": False} - for rom in self.exclude_single_files(fs_single_roms) - ] + [ - {"fs_name": rom, "flat": False, "nested": True} - for rom in self.exclude_multi_roms(fs_multi_roms) - ] - return sorted( [ FSRom( fs_name=rom["fs_name"], + fs_path=rom["fs_path"], flat=rom["flat"], nested=rom["nested"], files=[], @@ -719,7 +795,7 @@ async def get_roms(self, platform: Platform) -> list[FSRom]: ) for rom in fs_roms ], - key=lambda rom: rom["fs_name"], + key=lambda rom: (rom["fs_path"], rom["fs_name"]), ) async def rename_fs_rom(self, old_name: str, new_name: str, fs_path: str) -> None: diff --git a/backend/tests/endpoints/sockets/test_scan.py b/backend/tests/endpoints/sockets/test_scan.py index abac05e2a9..0dfd2ee52c 100644 --- a/backend/tests/endpoints/sockets/test_scan.py +++ b/backend/tests/endpoints/sockets/test_scan.py @@ -1,12 +1,21 @@ +import hashlib +from pathlib import Path from unittest.mock import Mock import pytest import socketio -from endpoints.sockets.scan import ScanStats, _should_scan_rom -from handler.filesystem.roms_handler import FSRomsHandler +from endpoints.sockets.scan import ( + ScanStats, + _reconcile_relocated_roms, + _should_scan_rom, +) +from handler.database import db_rom_handler +from handler.filesystem import fs_rom_handler +from handler.filesystem.roms_handler import FSRom, FSRomsHandler from handler.metadata.base_handler import UniversalPlatformSlug as UPS from handler.scan_handler import ScanType +from models.platform import Platform from models.rom import Rom @@ -326,3 +335,97 @@ def test_url_contains_fs_path_and_name(self, handler: FSRomsHandler): assert url is not None assert fs_path in url assert fs_name in url + + +def _fs_rom(fs_name: str, fs_path: str) -> FSRom: + return FSRom( + fs_name=fs_name, + fs_path=fs_path, + flat=True, + nested=False, + files=[], + crc_hash="", + md5_hash="", + sha1_hash="", + ra_hash="", + ) + + +class TestReconcileRelocatedRoms: + """A rom whose on-disk path changed (subfolder scanning) is relocated in + place by content hash instead of being re-imported as new, so its DB row — + and the saves/history/favorites/collections attached to it — survives.""" + + @pytest.mark.asyncio + async def test_moved_file_is_relocated_not_reimported( + self, platform: Platform, tmp_path: Path, monkeypatch + ): + content = b"relocate me please" + sha1 = hashlib.sha1(content, usedforsecurity=False).hexdigest() + base = f"{platform.fs_slug}/roms" + + # An existing rom recorded at the platform root, with its content hash. + rom = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="Mover", + slug="mover", + fs_name="Mover.bin", + fs_path=base, + sha1_hash=sha1, + fs_size_bytes=len(content), + ) + ) + + # On disk the file now lives inside a subfolder (same bytes). + sub = tmp_path / base / "Hacks" + sub.mkdir(parents=True) + (sub / "Mover.bin").write_bytes(content) + monkeypatch.setattr(fs_rom_handler, "base_path", tmp_path) + + fs_roms = [_fs_rom("Mover.bin", f"{base}/Hacks")] + handled = await _reconcile_relocated_roms(platform, fs_roms) + + # The moved file is reported handled, so the scan loop skips it. + assert handled == {f"{base}/Hacks/Mover.bin"} + + # Same row, new path, present again — not a fresh import. + all_roms = db_rom_handler.get_roms_for_relocation(platform.id) + assert len(all_roms) == 1 + updated = db_rom_handler.get_rom(rom.id) + assert updated is not None + assert updated.id == rom.id + assert updated.fs_path == f"{base}/Hacks" + assert updated.fs_name == "Mover.bin" + assert updated.missing_from_fs is False + + @pytest.mark.asyncio + async def test_no_hash_match_is_left_for_normal_import( + self, platform: Platform, tmp_path: Path, monkeypatch + ): + base = f"{platform.fs_slug}/roms" + # Existing rom whose stored hash won't match anything on disk. + db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="Other", + slug="other", + fs_name="Other.bin", + fs_path=base, + sha1_hash=hashlib.sha1(b"unrelated", usedforsecurity=False).hexdigest(), + fs_size_bytes=len(b"unrelated"), + ) + ) + + content = b"a brand new game entirely" + sub = tmp_path / base / "Hacks" + sub.mkdir(parents=True) + (sub / "New.bin").write_bytes(content) + monkeypatch.setattr(fs_rom_handler, "base_path", tmp_path) + + handled = await _reconcile_relocated_roms( + platform, [_fs_rom("New.bin", f"{base}/Hacks")] + ) + + # Nothing relocated: the new file falls through to a normal import. + assert handled == set() diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index 575f002f80..536e14efd7 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -377,6 +377,190 @@ async def test_get_roms(self, handler: FSRomsHandler, platform, config): # Check excluded files are not present assert "excluded_test.tmp" not in rom_names + def _make_subfolder_config(self, subfolders: dict[str, bool | list[str]]) -> Config: + return Config( + EXCLUDED_PLATFORMS=[], + EXCLUDED_SINGLE_EXT=["tmp"], + EXCLUDED_SINGLE_FILES=[], + EXCLUDED_MULTI_FILES=["@eaDir"], + EXCLUDED_MULTI_PARTS_EXT=["tmp"], + EXCLUDED_MULTI_PARTS_FILES=[], + PLATFORMS_BINDING={}, + PLATFORMS_VERSIONS={}, + ROMS_FOLDER_NAME="roms", + FIRMWARE_FOLDER_NAME="bios", + SCAN_SUBFOLDERS=subfolders, + ) + + def _build_subfolder_library(self, tmp_path: Path, platform: Platform) -> None: + """Library with a flat rom, a grouping subfolder (with a nested + sub-subfolder and a basename colliding with the root), a hidden folder, + and a folder-style multi-file rom.""" + roms = tmp_path / platform.fs_slug / "roms" + roms.mkdir(parents=True) + (roms / "Game A.zip").write_text("a") + group = roms / "All but the Best" + group.mkdir() + (group / "Game A.zip").write_text("dup") # same basename, different folder + (group / "Hidden Gem.zip").write_text("c") + deeper = group / "deeper" + deeper.mkdir() + (deeper / "Way Down.zip").write_text("d") + hidden = roms / ".hidden" + hidden.mkdir() + (hidden / "disc.chd").write_text("x") + multi = roms / "Multi Disc Game" + multi.mkdir() + (multi / "disc1.bin").write_text("p") + + @pytest.mark.asyncio + async def test_get_roms_subfolders_disabled( + self, platform: Platform, tmp_path: Path + ): + """By default a subfolder is a single multi-file rom, not a group.""" + self._build_subfolder_library(tmp_path, platform) + handler = FSRomsHandler() + handler.base_path = tmp_path + + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_subfolder_config({}), + ): + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + base = f"{platform.fs_slug}/roms" + assert (base, "Game A.zip") in keys + # Directories surface as multi-file roms, not as groups. + assert (base, "All but the Best") in keys + assert (base, "Multi Disc Game") in keys + assert (base, ".hidden") in keys + # Nothing inside any subfolder is surfaced. + assert not any(r["fs_path"] != base for r in roms) + assert len(roms) == 4 + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_subfolders_enabled( + self, platform: Platform, tmp_path: Path + ): + """With subfolder scanning on, groups are descended into (recursively), + hidden folders are skipped, and identically-named files in different + folders stay distinct via their full path.""" + self._build_subfolder_library(tmp_path, platform) + handler = FSRomsHandler() + handler.base_path = tmp_path + + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_subfolder_config({platform.fs_slug: True}), + ): + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + base = f"{platform.fs_slug}/roms" + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (base, "Game A.zip") in keys + assert (f"{base}/All but the Best", "Game A.zip") in keys # collision kept + assert (f"{base}/All but the Best", "Hidden Gem.zip") in keys + assert (f"{base}/All but the Best/deeper", "Way Down.zip") in keys + # Group folders are descended into, so "Multi Disc Game" parts surface. + assert (f"{base}/Multi Disc Game", "disc1.bin") in keys + # Hidden folder is not descended into; it remains a multi-file rom. + assert (base, ".hidden") in keys + assert (f"{base}/.hidden", "disc.chd") not in keys + # All full-path keys are unique and count matches. + full_paths = [f"{r['fs_path']}/{r['fs_name']}" for r in roms] + assert len(full_paths) == len(set(full_paths)) + assert sum(1 for r in roms if r["fs_name"] == "Game A.zip") == 2 + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_subfolders_descriptor_dir_kept_whole( + self, platform: Platform, tmp_path: Path + ): + """A subfolder holding a disc/playlist descriptor (.m3u, .cue, ...) is a + multi-disc game: it stays a single multi-file rom even with subfolder + scanning on (not split per disc), while a plain grouping folder is still + recursed.""" + roms = tmp_path / platform.fs_slug / "roms" + roms.mkdir(parents=True) + (roms / "Flat Game.zip").write_text("a") + # Multi-disc game declared by an .m3u playlist -> one rom. + md = roms / "Final Fantasy VII" + md.mkdir() + (md / "disc1.chd").write_text("1") + (md / "disc2.chd").write_text("2") + (md / "Final Fantasy VII.m3u").write_text("disc1.chd\ndisc2.chd") + # Multi-file game declared by a .cue descriptor (cue+bin) -> one rom. + cue = roms / "Some CD Game" + cue.mkdir() + (cue / "Some CD Game.cue").write_text('FILE "Some CD Game.bin" BINARY') + (cue / "Some CD Game.bin").write_text("data") + # Plain grouping folder (no descriptor) -> recursed into. + grp = roms / "Hacks" + grp.mkdir() + (grp / "Hack A.zip").write_text("h") + + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_subfolder_config({platform.fs_slug: True}), + ): + roms_found = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + base = f"{platform.fs_slug}/roms" + keys = {(r["fs_path"], r["fs_name"]) for r in roms_found} + # The .m3u folder stays a single multi-file rom (not split per disc). + assert (base, "Final Fantasy VII") in keys + assert (f"{base}/Final Fantasy VII", "disc1.chd") not in keys + # The .cue+.bin folder stays whole too (the case raised in review). + assert (base, "Some CD Game") in keys + assert (f"{base}/Some CD Game", "Some CD Game.bin") not in keys + # A normal grouping folder is still recursed. + assert (f"{base}/Hacks", "Hack A.zip") in keys + assert (base, "Flat Game.zip") in keys + assert count == len(roms_found) + + @pytest.mark.asyncio + async def test_get_roms_subfolders_named_list( + self, platform: Platform, tmp_path: Path + ): + """A list value recurses only the named folders; every other folder + (including a folder-based multi-file game) stays a single multi-file + rom, and a non-named folder nested inside a named one is not split.""" + self._build_subfolder_library(tmp_path, platform) + handler = FSRomsHandler() + handler.base_path = tmp_path + + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_subfolder_config( + {platform.fs_slug: ["All but the Best"]} + ), + ): + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + base = f"{platform.fs_slug}/roms" + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + # Named folder is recursed -> its files become individual roms. + assert (f"{base}/All but the Best", "Game A.zip") in keys + assert (f"{base}/All but the Best", "Hidden Gem.zip") in keys + # A non-named folder nested inside it is kept whole (not recursed). + assert (f"{base}/All but the Best", "deeper") in keys + assert (f"{base}/All but the Best/deeper", "Way Down.zip") not in keys + # Folders NOT in the list stay as single multi-file roms. + assert (base, "Multi Disc Game") in keys + assert (f"{base}/Multi Disc Game", "disc1.bin") not in keys + # Top-level flat file and hidden folder behave as usual. + assert (base, "Game A.zip") in keys + assert (base, ".hidden") in keys + assert count == len(roms) + @pytest.mark.asyncio async def test_get_rom_files_single_rom( self, handler: FSRomsHandler, rom_single, config @@ -738,19 +922,19 @@ async def test_top_level_files_only_in_main_hash( # The main ROM hash should be different from the translation file hash # (this verifies that the translation is not included in the main hash) - assert ( - parsed_rom_files.md5_hash == base_game_rom_file.md5_hash - ), "Main ROM hash should include base game file" - assert ( - parsed_rom_files.md5_hash != translation_rom_file.md5_hash - ), "Main ROM hash should not include translation file" + assert parsed_rom_files.md5_hash == base_game_rom_file.md5_hash, ( + "Main ROM hash should include base game file" + ) + assert parsed_rom_files.md5_hash != translation_rom_file.md5_hash, ( + "Main ROM hash should not include translation file" + ) - assert ( - parsed_rom_files.sha1_hash == base_game_rom_file.sha1_hash - ), "Main ROM hash should include base game file" - assert ( - parsed_rom_files.sha1_hash != translation_rom_file.sha1_hash - ), "Main ROM hash should not include translation file" + assert parsed_rom_files.sha1_hash == base_game_rom_file.sha1_hash, ( + "Main ROM hash should include base game file" + ) + assert parsed_rom_files.sha1_hash != translation_rom_file.sha1_hash, ( + "Main ROM hash should not include translation file" + ) @pytest.mark.asyncio async def test_get_rom_files_with_chd_v5_uses_internal_hash( @@ -797,9 +981,9 @@ async def test_get_rom_files_with_chd_v5_uses_internal_hash( assert len(parsed_rom_files.rom_files) == 1 assert parsed_rom_files.crc_hash != "", "CRC should be computed from raw bytes" assert parsed_rom_files.md5_hash != "", "MD5 should be computed from raw bytes" - assert ( - parsed_rom_files.sha1_hash != "" - ), "SHA1 should be computed from raw bytes" + assert parsed_rom_files.sha1_hash != "", ( + "SHA1 should be computed from raw bytes" + ) # Raw file SHA1 is NOT the header SHA1 assert parsed_rom_files.sha1_hash != internal_sha1 @@ -1077,15 +1261,15 @@ async def test_get_rom_files_with_non_v5_chd_fallback_to_std_hashing( # All hashes should be populated (calculated from file content) assert len(parsed_rom_files.rom_files) == 1 - assert ( - parsed_rom_files.crc_hash != "" - ), "CRC hash should be calculated for non-v5 CHD" - assert ( - parsed_rom_files.md5_hash != "" - ), "MD5 hash should be calculated for non-v5 CHD" - assert ( - parsed_rom_files.sha1_hash != "" - ), "SHA1 hash should be calculated for non-v5 CHD" + assert parsed_rom_files.crc_hash != "", ( + "CRC hash should be calculated for non-v5 CHD" + ) + assert parsed_rom_files.md5_hash != "", ( + "MD5 hash should be calculated for non-v5 CHD" + ) + assert parsed_rom_files.sha1_hash != "", ( + "SHA1 hash should be calculated for non-v5 CHD" + ) # Verify they're actual hash values (not from an internal header) assert parsed_rom_files.rom_files[0].crc_hash == parsed_rom_files.crc_hash @@ -1498,9 +1682,9 @@ def test_extract_chd_hash_off_by_one_header_sizes(self, tmp_path): result = extract_chd_hash(chd_file) - assert ( - result == expected - ), f"Failed for size {size}: got {result}, expected {expected}" + assert result == expected, ( + f"Failed for size {size}: got {result}, expected {expected}" + ) def test_extract_chd_hash_corrupted_header_data(self, tmp_path): """Test handling of corrupted/invalid data in header fields""" diff --git a/backend/tests/handler/test_db_handler.py b/backend/tests/handler/test_db_handler.py index 35e89d7a67..932dc058dc 100644 --- a/backend/tests/handler/test_db_handler.py +++ b/backend/tests/handler/test_db_handler.py @@ -1,6 +1,7 @@ from datetime import datetime, timezone import pytest +from sqlalchemy import inspect as sa_inspect from sqlalchemy.exc import IntegrityError from config import ROMM_DB_DRIVER @@ -544,6 +545,38 @@ def test_bulk_mark_present_chunking(platform: Platform): assert updated.missing_from_fs is False +def test_get_roms_by_fs_name_keys_on_full_path(platform: Platform): + """Identically-named files in different subfolders must stay distinct: the + result is keyed on full path (fs_path/fs_name), not just the file name.""" + root = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="dup_root", + slug="dup-root", + fs_name="Game.zip", + fs_path=f"{platform.slug}/roms", + ) + ) + nested = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="dup_nested", + slug="dup-nested", + fs_name="Game.zip", + fs_path=f"{platform.slug}/roms/Hacks", + ) + ) + + result = db_rom_handler.get_roms_by_fs_name(platform.id, ["Game.zip"]) + + assert set(result) == { + f"{platform.slug}/roms/Game.zip", + f"{platform.slug}/roms/Hacks/Game.zip", + } + assert result[f"{platform.slug}/roms/Game.zip"].id == root.id + assert result[f"{platform.slug}/roms/Hacks/Game.zip"].id == nested.id + + def test_mark_missing_roms_small_platform(platform: Platform): """mark_missing_roms correctly identifies missing ROMs with a small keep list.""" rom_a = db_rom_handler.add_rom( @@ -583,12 +616,22 @@ def test_mark_missing_roms_small_platform(platform: Platform): ) ) - # Keep only rom_a and rom_c - missing = db_rom_handler.mark_missing_roms(platform.id, ["rom_a.zip", "rom_c.zip"]) + # Keep only rom_a and rom_c (keep-list holds full paths) + missing = db_rom_handler.mark_missing_roms( + platform.id, + [f"{platform.slug}/roms/rom_a.zip", f"{platform.slug}/roms/rom_c.zip"], + ) assert len(missing) == 1 assert missing[0].fs_name == "rom_b.zip" + # Regression: returned (detached) instances must carry `fs_path` so callers + # can read `rom.full_path` after the session closes without a + # DetachedInstanceError (the missing-rom warning in scan.py does exactly + # that). + assert "fs_path" not in sa_inspect(missing[0]).unloaded + assert missing[0].full_path == f"{platform.slug}/roms/rom_b.zip" + updated_b = db_rom_handler.get_rom(rom_b.id) assert updated_b is not None assert updated_b.missing_from_fs is True @@ -627,7 +670,9 @@ def test_mark_missing_roms_large_platform(platform: Platform): # Build a large keep list to verify mark_missing_roms() handles many entries. # Only rom_present.zip actually exists in DB; the rest are just filler. - fs_roms_to_keep = ["rom_present.zip"] + [f"filler_{i}.zip" for i in range(501)] + fs_roms_to_keep = [f"{platform.slug}/roms/rom_present.zip"] + [ + f"filler_{i}.zip" for i in range(501) + ] missing = db_rom_handler.mark_missing_roms(platform.id, fs_roms_to_keep) @@ -662,7 +707,7 @@ def test_mark_missing_roms_large_platform_all_present(platform: Platform): roms.append(rom) # Keep list has all real ROMs plus filler to exceed 500 - fs_roms_to_keep = [f"rom_{i}.zip" for i in range(3)] + [ + fs_roms_to_keep = [f"{platform.slug}/roms/rom_{i}.zip" for i in range(3)] + [ f"filler_{i}.zip" for i in range(500) ] diff --git a/backend/tests/handler/test_fastapi.py b/backend/tests/handler/test_fastapi.py index 1e240e542f..d9d6eceebf 100644 --- a/backend/tests/handler/test_fastapi.py +++ b/backend/tests/handler/test_fastapi.py @@ -70,6 +70,7 @@ async def test_scan_rom(): rom=rom, fs_rom={ "fs_name": "Paper Mario (USA).z64", + "fs_path": "n64/Paper Mario (USA)", "flat": True, "nested": False, "files": [ @@ -160,6 +161,7 @@ async def test_scan_rom_complete_clears_unselected_metadata( rom=rom, fs_rom={ "fs_name": "Paper Mario (USA).z64", + "fs_path": "n64/Paper Mario (USA)", "flat": True, "nested": False, "files": [], @@ -230,6 +232,7 @@ async def test_scan_rom_unmatched_fetches_ra_when_id_set_but_no_metadata( rom=rom, fs_rom={ "fs_name": "Jak and Daxter.chd", + "fs_path": "ps2", "flat": True, "nested": False, "files": [], @@ -292,6 +295,7 @@ async def test_scan_rom_unmatched_skips_ra_when_id_and_metadata_exist( rom=rom, fs_rom={ "fs_name": "Jak and Daxter.chd", + "fs_path": "ps2", "flat": True, "nested": False, "files": [], diff --git a/examples/config.example.yml b/examples/config.example.yml index 48a8968134..e379769225 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -145,6 +145,38 @@ # image: screenshot # Use as the tag in the exported gamelist.xml # pegasus: # export: false # Whether to export metadata.pegasus.txt for Pegasus +# # Recurse into platform subfolders, treating nested files as their own +# # ROMs rather than collapsing each subfolder into a single multi-file ROM. +# # Opt-in per platform. The value is either: +# # - true -> recurse EVERY subfolder of the platform; +# # - a list of folder names -> recurse ONLY those folders, leaving every +# # other folder as a single multi-file ROM (so folder-based multi-file +# # games elsewhere on the platform stay intact — the recommended form +# # when a platform mixes grouping folders with multi-file games); +# # - false / omitted -> default behavior (a subfolder is one multi-file ROM). +# # +# # Behavior & notes (read before enabling): +# # - Hidden (dot-prefixed) folders are never descended into. +# # - A subfolder containing a disc/playlist descriptor (.m3u, .cue, .gdi, +# # .ccd, .toc) is kept whole as a single multi-file ROM (a multi-disc +# # game) and is NOT split into one ROM per disc, even when recursed. +# # - With `true`, a multi-file game stored as a bare folder with no such +# # descriptor WOULD be split into separate ROMs. Use the named-list form +# # (don't list that game's folder) to keep it whole. +# # - Identity is content-based on scan: moving (or renaming) a ROM between +# # subfolders is detected by its hash and relocated in place, so its +# # saves, play history, favorites and collection membership follow it — +# # as long as hashing is enabled and the platform is hashable. When a +# # folder previously scanned as one multi-file ROM is now recursed, that +# # old entry is marked "missing from filesystem" (the scan log flags it); +# # delete the stale entry to clean up. +# # - Two files with the same name in different subfolders become distinct +# # ROMs; gamelist.xml metadata matching is by filename, so both may match +# # the same gamelist entry. +# subfolders: +# nes: true # recurse every NES subfolder +# snes: ["Hacks", "Translations"] # recurse only these; keep other folders whole +# megadrive: false # EmulatorJS per-core options # emulatorjs: diff --git a/frontend/src/locales/bg_BG/rom.json b/frontend/src/locales/bg_BG/rom.json index 7b8ff996b8..686ca331a7 100644 --- a/frontend/src/locales/bg_BG/rom.json +++ b/frontend/src/locales/bg_BG/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Направи публична", "screenshot-make-private": "Направи лична", "screenshot-public": "Публична", - "screenshot-visibility-failed": "Видимостта не може да се промени: {error}" + "screenshot-visibility-failed": "Видимостта не може да се промени: {error}", + "location": "Местоположение", + "location-copied": "Местоположението е копирано в клипборда." } diff --git a/frontend/src/locales/cs_CZ/rom.json b/frontend/src/locales/cs_CZ/rom.json index eeb05145e4..3265b15431 100644 --- a/frontend/src/locales/cs_CZ/rom.json +++ b/frontend/src/locales/cs_CZ/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Zveřejnit", "screenshot-make-private": "Nastavit jako soukromé", "screenshot-public": "Veřejné", - "screenshot-visibility-failed": "Nepodařilo se změnit viditelnost: {error}" + "screenshot-visibility-failed": "Nepodařilo se změnit viditelnost: {error}", + "location": "Umístění", + "location-copied": "Umístění zkopírováno do schránky." } diff --git a/frontend/src/locales/de_DE/rom.json b/frontend/src/locales/de_DE/rom.json index 9b297e2af1..9af68a48b2 100644 --- a/frontend/src/locales/de_DE/rom.json +++ b/frontend/src/locales/de_DE/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Veröffentlichen", "screenshot-make-private": "Privat machen", "screenshot-public": "Öffentlich", - "screenshot-visibility-failed": "Sichtbarkeit konnte nicht geändert werden: {error}" + "screenshot-visibility-failed": "Sichtbarkeit konnte nicht geändert werden: {error}", + "location": "Speicherort", + "location-copied": "Speicherort in die Zwischenablage kopiert." } diff --git a/frontend/src/locales/en_GB/rom.json b/frontend/src/locales/en_GB/rom.json index 7fc0046cf1..98613e1865 100644 --- a/frontend/src/locales/en_GB/rom.json +++ b/frontend/src/locales/en_GB/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Make public", "screenshot-make-private": "Make private", "screenshot-public": "Public", - "screenshot-visibility-failed": "Couldn't update visibility: {error}" + "screenshot-visibility-failed": "Couldn't update visibility: {error}", + "location": "Location", + "location-copied": "Location copied to clipboard." } diff --git a/frontend/src/locales/en_US/rom.json b/frontend/src/locales/en_US/rom.json index 371f3a417d..65f1ad6e1a 100644 --- a/frontend/src/locales/en_US/rom.json +++ b/frontend/src/locales/en_US/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Make public", "screenshot-make-private": "Make private", "screenshot-public": "Public", - "screenshot-visibility-failed": "Couldn't update visibility: {error}" + "screenshot-visibility-failed": "Couldn't update visibility: {error}", + "location": "Location", + "location-copied": "Location copied to clipboard." } diff --git a/frontend/src/locales/es_ES/rom.json b/frontend/src/locales/es_ES/rom.json index b03eac2140..ab632711ae 100644 --- a/frontend/src/locales/es_ES/rom.json +++ b/frontend/src/locales/es_ES/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Hacer pública", "screenshot-make-private": "Hacer privada", "screenshot-public": "Pública", - "screenshot-visibility-failed": "No se pudo actualizar la visibilidad: {error}" + "screenshot-visibility-failed": "No se pudo actualizar la visibilidad: {error}", + "location": "Ubicación", + "location-copied": "Ubicación copiada al portapapeles." } diff --git a/frontend/src/locales/fr_FR/rom.json b/frontend/src/locales/fr_FR/rom.json index 15b40bfd10..08e2c33839 100644 --- a/frontend/src/locales/fr_FR/rom.json +++ b/frontend/src/locales/fr_FR/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Rendre publique", "screenshot-make-private": "Rendre privée", "screenshot-public": "Publique", - "screenshot-visibility-failed": "Impossible de mettre à jour la visibilité : {error}" + "screenshot-visibility-failed": "Impossible de mettre à jour la visibilité : {error}", + "location": "Emplacement", + "location-copied": "Emplacement copié dans le presse-papiers." } diff --git a/frontend/src/locales/hu_HU/rom.json b/frontend/src/locales/hu_HU/rom.json index 75bd5ad406..120e8ba081 100644 --- a/frontend/src/locales/hu_HU/rom.json +++ b/frontend/src/locales/hu_HU/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Közzététel", "screenshot-make-private": "Priváttá tétel", "screenshot-public": "Nyilvános", - "screenshot-visibility-failed": "A láthatóság nem frissíthető: {error}" + "screenshot-visibility-failed": "A láthatóság nem frissíthető: {error}", + "location": "Hely", + "location-copied": "A hely a vágólapra másolva." } diff --git a/frontend/src/locales/it_IT/rom.json b/frontend/src/locales/it_IT/rom.json index ff548e326e..0ae27c9dd2 100644 --- a/frontend/src/locales/it_IT/rom.json +++ b/frontend/src/locales/it_IT/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Rendi pubblico", "screenshot-make-private": "Rendi privato", "screenshot-public": "Pubblico", - "screenshot-visibility-failed": "Impossibile aggiornare la visibilità: {error}" + "screenshot-visibility-failed": "Impossibile aggiornare la visibilità: {error}", + "location": "Posizione", + "location-copied": "Posizione copiata negli appunti." } diff --git a/frontend/src/locales/ja_JP/rom.json b/frontend/src/locales/ja_JP/rom.json index 7f5b920249..adcfed7dfd 100644 --- a/frontend/src/locales/ja_JP/rom.json +++ b/frontend/src/locales/ja_JP/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "公開する", "screenshot-make-private": "非公開にする", "screenshot-public": "公開", - "screenshot-visibility-failed": "表示設定を更新できませんでした：{error}" + "screenshot-visibility-failed": "表示設定を更新できませんでした：{error}", + "location": "場所", + "location-copied": "場所をクリップボードにコピーしました。" } diff --git a/frontend/src/locales/ko_KR/rom.json b/frontend/src/locales/ko_KR/rom.json index d461dda5a4..45196dfd0d 100644 --- a/frontend/src/locales/ko_KR/rom.json +++ b/frontend/src/locales/ko_KR/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "공개로 전환", "screenshot-make-private": "비공개로 전환", "screenshot-public": "공개", - "screenshot-visibility-failed": "공개 설정을 변경할 수 없습니다: {error}" + "screenshot-visibility-failed": "공개 설정을 변경할 수 없습니다: {error}", + "location": "위치", + "location-copied": "위치를 클립보드에 복사했습니다." } diff --git a/frontend/src/locales/pl_PL/rom.json b/frontend/src/locales/pl_PL/rom.json index ef472148c3..6fc1d78447 100644 --- a/frontend/src/locales/pl_PL/rom.json +++ b/frontend/src/locales/pl_PL/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Upublicznij", "screenshot-make-private": "Ustaw jako prywatne", "screenshot-public": "Publiczne", - "screenshot-visibility-failed": "Nie udało się zaktualizować widoczności: {error}" + "screenshot-visibility-failed": "Nie udało się zaktualizować widoczności: {error}", + "location": "Lokalizacja", + "location-copied": "Skopiowano lokalizację do schowka." } diff --git a/frontend/src/locales/pt_BR/rom.json b/frontend/src/locales/pt_BR/rom.json index 349510ad46..7550d3e9d1 100644 --- a/frontend/src/locales/pt_BR/rom.json +++ b/frontend/src/locales/pt_BR/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Tornar pública", "screenshot-make-private": "Tornar privada", "screenshot-public": "Pública", - "screenshot-visibility-failed": "Não foi possível atualizar a visibilidade: {error}" + "screenshot-visibility-failed": "Não foi possível atualizar a visibilidade: {error}", + "location": "Localização", + "location-copied": "Localização copiada para a área de transferência." } diff --git a/frontend/src/locales/ro_RO/rom.json b/frontend/src/locales/ro_RO/rom.json index c56dac717e..48bf7f9afb 100644 --- a/frontend/src/locales/ro_RO/rom.json +++ b/frontend/src/locales/ro_RO/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Fă publică", "screenshot-make-private": "Fă privată", "screenshot-public": "Publică", - "screenshot-visibility-failed": "Vizibilitatea nu a putut fi actualizată: {error}" + "screenshot-visibility-failed": "Vizibilitatea nu a putut fi actualizată: {error}", + "location": "Locație", + "location-copied": "Locație copiată în clipboard." } diff --git a/frontend/src/locales/ru_RU/rom.json b/frontend/src/locales/ru_RU/rom.json index 045aeecd93..eb3aae403f 100644 --- a/frontend/src/locales/ru_RU/rom.json +++ b/frontend/src/locales/ru_RU/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "Сделать публичной", "screenshot-make-private": "Сделать приватной", "screenshot-public": "Публичная", - "screenshot-visibility-failed": "Не удалось изменить видимость: {error}" + "screenshot-visibility-failed": "Не удалось изменить видимость: {error}", + "location": "Расположение", + "location-copied": "Расположение скопировано в буфер обмена." } diff --git a/frontend/src/locales/zh_CN/rom.json b/frontend/src/locales/zh_CN/rom.json index 877a90fa90..a7c88b0367 100644 --- a/frontend/src/locales/zh_CN/rom.json +++ b/frontend/src/locales/zh_CN/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "设为公开", "screenshot-make-private": "设为私密", "screenshot-public": "公开", - "screenshot-visibility-failed": "无法更新可见性：{error}" + "screenshot-visibility-failed": "无法更新可见性：{error}", + "location": "位置", + "location-copied": "位置已复制到剪贴板。" } diff --git a/frontend/src/locales/zh_TW/rom.json b/frontend/src/locales/zh_TW/rom.json index 0bdec716f1..6223181236 100644 --- a/frontend/src/locales/zh_TW/rom.json +++ b/frontend/src/locales/zh_TW/rom.json @@ -405,5 +405,7 @@ "screenshot-make-public": "設為公開", "screenshot-make-private": "設為私密", "screenshot-public": "公開", - "screenshot-visibility-failed": "無法更新可見性：{error}" + "screenshot-visibility-failed": "無法更新可見性：{error}", + "location": "位置", + "location-copied": "位置已複製到剪貼簿。" } diff --git a/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue b/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue index 71beee439d..5518a518b4 100644 --- a/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue +++ b/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue @@ -7,6 +7,7 @@ import { computed } from "vue"; import type { DetailedRomSchema } from "@/__generated__"; import { formatBytes } from "@/utils"; import HashChip from "@/v2/components/shared/HashChip.vue"; +import LocationChip from "@/v2/components/shared/LocationChip.vue"; import MissingFSBadge from "@/v2/components/shared/MissingFSBadge.vue"; defineOptions({ inheritAttrs: false }); @@ -15,6 +16,12 @@ const props = defineProps<{ rom: DetailedRomSchema }>(); const fileCount = computed(() => props.rom.files?.length ?? 0); +// Full on-disk path of the ROM — its directory plus its name. Surfaces +// *where* a ROM sits in the library (e.g. a platform subfolder like +// `roms/nes/Hacks/…`), which is otherwise only implicit. Distinct from +// the per-file relative paths shown in the file list below. +const fullPath = computed(() => `${props.rom.fs_path}/${props.rom.fs_name}`); + interface RomHash { label: string; value: string | null; @@ -57,6 +64,10 @@ const hashes = computed(() => { +
+ +
+
(() => { .r-v2-files-summary__sep { opacity: 0.5; } +.r-v2-files-summary__location { + display: flex; + min-width: 0; +} .r-v2-files-summary__hashes { display: flex; align-items: center; diff --git a/frontend/src/v2/components/shared/LocationChip.vue b/frontend/src/v2/components/shared/LocationChip.vue new file mode 100644 index 0000000000..5f1d5414c7 --- /dev/null +++ b/frontend/src/v2/components/shared/LocationChip.vue @@ -0,0 +1,82 @@ + + + + + From b5f457b53a2d4cfddd1208b5232d2c8151e2322d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 22:01:05 +0000 Subject: [PATCH 2/4] feat: replace subfolder flag with per-platform custom library structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalizes the opt-in subfolder scanning into a per-platform custom library structure template, modeled on Retrom's. Replaces the `scan.subfolders` (bool | list) flag with `filesystem.structure` (fs_slug -> template); platforms without a template keep RomM's default discovery (top-level files and folders), so existing setups are unaffected. A template is relative to the platform's ROM folder (RomM already resolves the library root, roms_folder and platform directory) and is a sequence of `/`-separated path sections: - a braced section is a macro, a bare section is a literal folder matched exactly; - the last section must be the terminal {gameFile} (each file is a game) or {gameDir} (each folder is a single multi-file game); - any other braced section ({region}, {category}, ...) is a wildcard directory level that matches any folder (organizational only). Examples: filesystem: structure: nes: "{category}/{gameFile}" # roms/nes/Hacks/foo.nes ps3: "{category}/{gameDir}" # roms/ps3/PSN/Game/ snes: "{region}/{gameFile}" # roms/snes/USA/foo.sfc Declaring {gameFile} vs {gameDir} removes the previous disc-descriptor guessing entirely — the user states file-vs-folder. Hidden (dot-prefixed) folders are never descended into. Hash-based non-destructive identity (relocate a moved/renamed game in place, preserving saves/history/favorites/ collections) is retained and now triggers for any platform with a custom structure. - config: parse + validate templates at load (parse_library_structure, LibraryStructure); reject {platform}/{library} (RomM resolves those) and malformed templates. - fs handler: _discover_structured_roms walks literal/wildcard levels to the terminal; _discover_default_roms preserves the default behavior. - Tests: template parser (valid/invalid), per-platform config loading, and structured discovery (wildcard/literal levels, file/dir terminals, depth>1, hidden-folder skip, cross-folder name collisions). Subfolder-flag tests replaced. - Docs: config.example.yml documents filesystem.structure; scan.subfolders removed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Sk2dzM3K9qxWBdPAeGb7us --- backend/config/config_manager.py | 153 +++++++++--- backend/endpoints/sockets/scan.py | 30 +-- backend/handler/database/roms_handler.py | 8 +- backend/handler/filesystem/roms_handler.py | 148 ++++++------ .../tests/config/fixtures/config/config.yml | 3 + backend/tests/config/test_config_loader.py | 51 ++++ backend/tests/endpoints/sockets/test_scan.py | 8 +- .../handler/filesystem/test_roms_handler.py | 219 +++++++++--------- backend/tests/handler/test_db_handler.py | 2 +- examples/config.example.yml | 70 +++--- 10 files changed, 422 insertions(+), 270 deletions(-) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 91edc46267..1e414ba0b2 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -4,6 +4,7 @@ import json import os import sys +from dataclasses import dataclass from pathlib import Path from typing import Final, NotRequired, TypedDict @@ -28,6 +29,98 @@ from logger.formatter import highlight as hl from logger.logger import log + +# Terminal macros for a custom library structure template. Exactly one must +# appear, as the last path segment. +STRUCTURE_GAME_FILE: Final = "gameFile" +STRUCTURE_GAME_DIR: Final = "gameDir" +# Macros that belong to Retrom's full-path templates but are resolved by RomM +# itself (library base + roms folder + platform discovery). A per-platform +# template is relative to the platform's ROM folder, so these are rejected with +# a helpful message instead of being silently treated as wildcard levels. +_STRUCTURE_RESERVED_NONTERMINAL: Final = frozenset({"platform", "library"}) + + +@dataclass(frozen=True) +class StructureLevel: + """One intermediate directory level of a custom structure template. + + ``literal`` is the exact folder name to match, or ``None`` for a wildcard + macro level (``{region}``, ``{category}``, …) that matches any folder. + """ + + literal: str | None + + +@dataclass(frozen=True) +class LibraryStructure: + """A parsed per-platform custom library structure. + + Relative to the platform's ROM folder: ``levels`` are the intermediate + directory levels to descend (literal or wildcard), and ``each_file_is_game`` + selects the terminal — ``{gameFile}`` (each file is a ROM) vs ``{gameDir}`` + (each directory is a multi-file ROM). + """ + + levels: tuple[StructureLevel, ...] + each_file_is_game: bool + + +def parse_library_structure(template: str) -> LibraryStructure: + """Parse a custom structure template into a ``LibraryStructure``. + + Template syntax mirrors Retrom: ``/``-separated path sections where a + section wrapped in braces is a macro and a bare section is a literal folder + name. The last section must be the terminal ``{gameFile}`` or ``{gameDir}``; + every other braced section is a wildcard directory level. The template is + relative to the platform's ROM folder, so ``{platform}`` / ``{library}`` are + not used here. + + Raises ``ValueError`` on an invalid template. + """ + sections = [s for s in template.split("/") if s != ""] + if not sections: + raise ValueError("template is empty") + + levels: list[StructureLevel] = [] + each_file_is_game: bool | None = None + + for index, section in enumerate(sections): + is_last = index == len(sections) - 1 + is_macro = section.startswith("{") and section.endswith("}") + + if not is_macro: + if "{" in section or "}" in section: + raise ValueError(f"malformed macro in section '{section}'") + levels.append(StructureLevel(literal=section)) + continue + + name = section[1:-1].strip() + if name in (STRUCTURE_GAME_FILE, STRUCTURE_GAME_DIR): + if not is_last: + raise ValueError( + f"'{{{name}}}' must be the last path section of the template" + ) + each_file_is_game = name == STRUCTURE_GAME_FILE + continue + + if name in _STRUCTURE_RESERVED_NONTERMINAL: + raise ValueError( + f"'{{{name}}}' is not supported: a platform template is relative " + "to that platform's ROM folder (RomM resolves library root, roms " + "folder and platform on its own)" + ) + if not name: + raise ValueError("empty macro '{}'") + # Any other braced section is an organizational wildcard directory level. + levels.append(StructureLevel(literal=None)) + + if each_file_is_game is None: + raise ValueError("template must end with '{gameFile}' or '{gameDir}'") + + return LibraryStructure(levels=tuple(levels), each_file_is_game=each_file_is_game) + + ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config" ROMM_USER_CONFIG_FILE: Final = f"{ROMM_USER_CONFIG_PATH}/config.yml" SQLITE_DB_BASE_PATH: Final = f"{ROMM_BASE_PATH}/database" @@ -118,6 +211,7 @@ class Config: PLATFORMS_VERSIONS: dict[str, str] ROMS_FOLDER_NAME: str FIRMWARE_FOLDER_NAME: str + STRUCTURE_TEMPLATES: dict[str, str] SKIP_HASH_CALCULATION: bool EJS_DEBUG: bool EJS_CACHE_LIMIT: int | None @@ -132,7 +226,6 @@ class Config: SCAN_REGION_PRIORITY: list[str] SCAN_LANGUAGE_PRIORITY: list[str] SCAN_MEDIA: list[str] - SCAN_SUBFOLDERS: dict[str, bool | list[str]] GAMELIST_MEDIA_THUMBNAIL: MetadataMediaType GAMELIST_MEDIA_IMAGE: MetadataMediaType @@ -161,24 +254,18 @@ def has_structure_path_b(self) -> bool: return False - def subfolder_scan_spec(self, fs_slug: str) -> bool | frozenset[str]: - """How the scanner should recurse into a platform's subfolders. - - Opt-in per platform via `scan.subfolders` in config.yml, where the - value is either: - - ``True`` -> recurse every subfolder (each nested file becomes its - own rom); - - a list of folder names -> recurse only those folders, leaving every - other folder as a single multi-file rom (so folder-based multi-file - games elsewhere on the platform stay intact); - - ``False`` / omitted -> don't recurse (default behavior). + def platform_structure(self, fs_slug: str) -> LibraryStructure | None: + """The custom library structure for a platform, or ``None``. - Returns ``True``, a ``frozenset`` of folder names, or ``False``. + Opt-in per platform via `filesystem.structure` in config.yml (a map of + ``fs_slug -> template``). When unset, the platform uses RomM's default + discovery (top-level files and folders). The template is validated at + load time, so parsing here is expected to succeed. """ - value = getattr(self, "SCAN_SUBFOLDERS", {}).get(fs_slug, False) - if isinstance(value, list): - return frozenset(value) - return bool(value) + template = getattr(self, "STRUCTURE_TEMPLATES", {}).get(fs_slug) + if not template: + return None + return parse_library_structure(template) class ConfigManager: @@ -466,7 +553,9 @@ def _parse_config(self): PEGASUS_AUTO_EXPORT_ON_SCAN=pydash.get( self._raw_config, "scan.pegasus.export", False ), - SCAN_SUBFOLDERS=pydash.get(self._raw_config, "scan.subfolders", {}), + STRUCTURE_TEMPLATES=pydash.get( + self._raw_config, "filesystem.structure", {} + ), ) def _get_ejs_controls(self) -> dict[str, EjsControls]: @@ -681,18 +770,24 @@ def _validate_config(self): log.critical("Invalid config.yml: scan.media must be a list") sys.exit(3) - if not isinstance(self.config.SCAN_SUBFOLDERS, dict): - log.critical("Invalid config.yml: scan.subfolders must be a dictionary") - sys.exit(3) - for fs_slug, value in self.config.SCAN_SUBFOLDERS.items(): - is_bool = isinstance(value, bool) - is_str_list = isinstance(value, list) and all( - isinstance(name, str) for name in value + if not isinstance(self.config.STRUCTURE_TEMPLATES, dict): + log.critical( + "Invalid config.yml: filesystem.structure must be a dictionary" ) - if not (is_bool or is_str_list): + sys.exit(3) + for fs_slug, template in self.config.STRUCTURE_TEMPLATES.items(): + if not isinstance(template, str): + log.critical( + f"Invalid config.yml: filesystem.structure.{fs_slug} must be a " + "string template" + ) + sys.exit(3) + try: + parse_library_structure(template) + except ValueError as exc: log.critical( - f"Invalid config.yml: scan.subfolders.{fs_slug} must be a " - "boolean or a list of folder names" + f"Invalid config.yml: filesystem.structure.{fs_slug} " + f"('{template}'): {exc}" ) sys.exit(3) @@ -789,6 +884,7 @@ def _update_config_file(self) -> None: "filesystem": { "roms_folder": self.config.ROMS_FOLDER_NAME, "firmware_folder": self.config.FIRMWARE_FOLDER_NAME, + "structure": self.config.STRUCTURE_TEMPLATES, "skip_hash_calculation": self.config.SKIP_HASH_CALCULATION, }, "system": { @@ -815,7 +911,6 @@ def _update_config_file(self) -> None: "language": self.config.SCAN_LANGUAGE_PRIORITY, }, "media": self.config.SCAN_MEDIA, - "subfolders": self.config.SCAN_SUBFOLDERS, "gamelist": { "export": self.config.GAMELIST_AUTO_EXPORT_ON_SCAN, "media": { diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index e12be729be..08cb4ac243 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -249,8 +249,8 @@ async def _identify_rom( # Update properties that don't require metadata parsed_tags = fs_rom_handler.parse_tags(fs_rom["fs_name"]) - # The discovered path is the rom's actual directory, which may be a - # subfolder of the platform roms folder when subfolder scanning is enabled. + # The discovered path is the rom's actual directory, which may be nested + # under the platform roms folder when a custom structure is configured. roms_path = fs_rom["fs_path"] # Create the entry early so we have the ID @@ -541,7 +541,7 @@ async def _reconcile_relocated_roms( ) -> set[str]: """Relocate roms whose on-disk path changed instead of re-importing them. - When subfolder scanning is enabled, moving (or renaming) a file reads as a + Under a custom library structure, moving (or renaming) a file reads as a new path. Rather than insert a fresh rom and mark the old one missing — which would drop saves, play history, favorites and collection membership — match a newly-seen file to a now-missing rom by content hash and update that @@ -562,7 +562,7 @@ async def _reconcile_relocated_roms( # Group candidates by total size: a moved file keeps its size, so this is a # cheap pre-filter that avoids hashing every newly-seen file (e.g. on first - # enable, where the whole subfolder tree reads as new). + # enable, where the whole structure reads as new). by_size: dict[int, list[Rom]] = defaultdict(list) for rom in disappeared: by_size[rom.fs_size_bytes].append(rom) @@ -739,12 +739,12 @@ async def _identify_platform( else: log.info(f"{hl(str(len(fs_roms)))} roms found in the file system") - # Detect roms that simply moved on disk (subfolder scanning) and relocate - # them in place so their saves/history/favorites/collections follow, - # instead of re-importing them as new and orphaning the old entry. Only - # runs when subfolder scanning is enabled for the platform, so default - # libraries pay no extra cost. - if cm.get_config().subfolder_scan_spec(platform.fs_slug): + # Detect roms that simply moved on disk (custom library structure) and + # relocate them in place so their saves/history/favorites/collections + # follow, instead of re-importing them as new and orphaning the old entry. + # Only runs for platforms with a custom structure, so default libraries pay + # no extra cost. + if cm.get_config().platform_structure(platform.fs_slug) is not None: relocated_paths = await _reconcile_relocated_roms(platform, fs_roms) if relocated_paths: await scan_stats.increment( @@ -778,8 +778,8 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: for fs_roms_batch in batched(fs_roms, 200, strict=False): # Key matches on the rom's full path (fs_path/fs_name), not just the - # file name, so identically-named files in different subfolders don't - # collide when subfolder scanning is enabled. + # file name, so identically-named files in different folders don't + # collide under a custom library structure. roms_by_full_path = db_rom_handler.get_roms_by_fs_name( platform_id=platform.id, fs_names={fs_rom["fs_name"] for fs_rom in fs_roms_batch}, @@ -826,15 +826,15 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: ) if len(missing_roms) > 0: log.warning(f"{hl('Missing')} roms from filesystem:") - # A folder that subfolder scanning now recurses into used to be a single + # A folder a custom structure now descends into used to be a single # multi-file rom; that old entry shows up here as missing. Flag those so # it's clear the "missing" is expected and the stale entry can be # deleted. A superseded folder's path is a parent of a discovered rom. - recursed_paths = {rom["fs_path"] for rom in fs_roms} + descended_paths = {rom["fs_path"] for rom in fs_roms} for r in missing_roms: superseded = any( p == r.full_path or p.startswith(f"{r.full_path}/") - for p in recursed_paths + for p in descended_paths ) if superseded: log.warning( diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index 34c4830eb5..9a2fe24e0c 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -1174,8 +1174,8 @@ def get_roms_by_fs_name( """Retrieve a dictionary of roms keyed by their full path (fs_path/fs_name). Filters by file name for an indexed lookup, but keys the result on the - full path so identically-named files in different subfolders (subfolder - scanning) remain distinct. + full path so identically-named files in different folders (custom + library structures) remain distinct. Eager-loads only `platform` (used downstream by the scan loop via `rom.platform_slug` / `rom.platform.fs_slug`). This deliberately @@ -1283,8 +1283,8 @@ def mark_missing_roms( """Sync `missing_from_fs` for a platform against the keep-list. The keep-list holds rom full paths (fs_path/fs_name) so that - identically-named files in different subfolders are tracked - independently when subfolder scanning is enabled. + identically-named files in different folders are tracked + independently under a custom library structure. Reads the rows once and writes only those whose state actually changes, so a re-scan of an unchanged platform issues no updates. diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index 4d5de00a7b..a407a6601e 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -15,6 +15,7 @@ from config.config_manager import ( DEFAULT_EXCLUDED_EXTENSIONS, DEFAULT_EXCLUDED_FILES, + LibraryStructure, ) from config.config_manager import config_manager as cm from exceptions.fs_exceptions import ( @@ -318,7 +319,7 @@ async def get_rom_files( from handler.metadata import meta_ra_handler # The rom's stored directory is the source of truth for its location, so - # roms inside a platform subfolder (subfolder scanning) resolve to their + # roms inside a nested folder (custom library structure) resolve to their # real path rather than the platform roms root. rel_roms_path = rom.fs_path # Relative path to the rom's directory abs_fs_path = self.validate_path(rel_roms_path) # Absolute path to that dir @@ -676,90 +677,91 @@ def update_hashes(chunk: bytes | bytearray): rom_sha1_h, ) - # Disc/playlist descriptors that mark a folder as a single multi-file game - # (a multi-disc title), so it is kept whole instead of being recursed into - # when subfolder scanning is enabled — preventing it from being split into - # one rom per disc. - _MULTI_DISC_DESCRIPTOR_EXTS = (".m3u", ".cue", ".gdi", ".ccd", ".toc") + async def _discover_default_roms(self, rel_roms_path: str) -> list[dict]: + """Default discovery: top-level files and folders of the roms path. - @staticmethod - def _should_recurse_dir(directory: str, recurse: bool | frozenset[str]) -> bool: - """Whether to descend into ``directory`` (collecting its contents as - individual roms) rather than treat it as one multi-file rom. - - Hidden (dot-prefixed) folders are never descended into. Otherwise - ``recurse`` is ``True`` (all folders), a set of folder names (only - those), or ``False`` (none). - """ - if directory.startswith("."): - return False - if isinstance(recurse, frozenset): - return directory in recurse - return recurse - - async def _is_multi_disc_dir(self, rel_dir_path: str) -> bool: - """True if a directory directly contains a disc/playlist descriptor. - - Such a directory is a single multi-file rom (a multi-disc game), so it - is kept whole instead of being recursed into when subfolder scanning is - enabled — preventing a multi-disc game from being split into one rom per - disc. Covers ``.m3u`` playlists as well as ``.cue``/``.gdi``/``.ccd``/ - ``.toc`` track descriptors (the ``.cue``+``.bin`` case raised in review). - """ - return any( - f.lower().endswith(self._MULTI_DISC_DESCRIPTOR_EXTS) - for f in await self.list_files(rel_dir_path) - ) - - async def _collect_fs_roms( - self, rel_roms_path: str, recurse: bool | frozenset[str] - ) -> list[dict]: - """Discover roms under ``rel_roms_path``. - - Single files are flat roms (``fs_path`` = their parent directory). A - directory is a single multi-file rom unless ``recurse`` says to descend - into it, in which case its contents are collected as individual roms. - ``recurse`` is ``True`` (descend every folder), a set of folder names - (descend only those — every other folder stays a multi-file rom), or - ``False`` (descend none). - - Hidden (dot-prefixed) directories are never descended into, and a - directory holding a disc/playlist descriptor is kept whole as a single - multi-file rom (a multi-disc game) even while recursing. + Each top-level file is a flat rom; each top-level directory is a single + multi-file rom. This is RomM's behavior for platforms without a custom + structure template. """ fs_roms: list[dict] = [ {"fs_name": rom, "fs_path": rel_roms_path, "flat": True, "nested": False} for rom in self.exclude_single_files(await self.list_files(rel_roms_path)) ] + fs_roms += [ + {"fs_name": rom, "fs_path": rel_roms_path, "flat": False, "nested": True} + for rom in self.exclude_multi_roms( + await self.list_directories(rel_roms_path) + ) + ] + return fs_roms - for directory in self.exclude_multi_roms( - await self.list_directories(rel_roms_path) - ): - dir_path = f"{rel_roms_path}/{directory}" - if self._should_recurse_dir( - directory, recurse - ) and not await self._is_multi_disc_dir(dir_path): - fs_roms.extend(await self._collect_fs_roms(dir_path, recurse)) + async def _discover_structured_roms( + self, rel_roms_path: str, structure: LibraryStructure + ) -> list[dict]: + """Discover roms following a custom library structure template. + + Descends the template's intermediate directory levels (literal names + matched exactly, wildcard macros matching any folder), then collects + roms at the terminal: ``{gameFile}`` makes each file a rom, ``{gameDir}`` + makes each directory a (multi-file) rom. Each rom records its real + ``fs_path``. Hidden (dot-prefixed) folders are never descended into or + surfaced. + """ + dirs = [rel_roms_path] + for level in structure.levels: + next_dirs: list[str] = [] + for directory in dirs: + for sub in await self.list_directories(directory): + if sub.startswith("."): + continue + if level.literal is not None and sub != level.literal: + continue + next_dirs.append(f"{directory}/{sub}") + dirs = next_dirs + + fs_roms: list[dict] = [] + for directory in dirs: + if structure.each_file_is_game: + for name in self.exclude_single_files(await self.list_files(directory)): + fs_roms.append( + { + "fs_name": name, + "fs_path": directory, + "flat": True, + "nested": False, + } + ) else: - fs_roms.append( - { - "fs_name": directory, - "fs_path": rel_roms_path, - "flat": False, - "nested": True, - } - ) - + for name in self.exclude_multi_roms( + await self.list_directories(directory) + ): + if name.startswith("."): + continue + fs_roms.append( + { + "fs_name": name, + "fs_path": directory, + "flat": False, + "nested": True, + } + ) return fs_roms + async def _collect_fs_roms(self, platform: Platform) -> list[dict]: + """Discover a platform's roms, honoring its custom structure if set.""" + rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) + structure = cm.get_config().platform_structure(platform.fs_slug) + if structure is None: + return await self._discover_default_roms(rel_roms_path) + return await self._discover_structured_roms(rel_roms_path, structure) + async def count_roms(self, platform: Platform) -> int: """Return the number of filesystem roms for a platform without materializing FSRom objects. """ - recurse = cm.get_config().subfolder_scan_spec(platform.fs_slug) try: - rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) - return len(await self._collect_fs_roms(rel_roms_path, recurse)) + return len(await self._collect_fs_roms(platform)) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e @@ -771,12 +773,8 @@ async def get_roms(self, platform: Platform) -> list[FSRom]: Returns: list with all the filesystem roms for a platform """ - recurse = cm.get_config().subfolder_scan_spec(platform.fs_slug) try: - rel_roms_path = self.get_roms_fs_structure( - platform.fs_slug - ) # Relative path to roms - fs_roms = await self._collect_fs_roms(rel_roms_path, recurse) + fs_roms = await self._collect_fs_roms(platform) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e diff --git a/backend/tests/config/fixtures/config/config.yml b/backend/tests/config/fixtures/config/config.yml index 1e7c22a0db..1bf6ecfb64 100644 --- a/backend/tests/config/fixtures/config/config.yml +++ b/backend/tests/config/fixtures/config/config.yml @@ -30,6 +30,9 @@ filesystem: roms_folder: "ROMS" firmware_folder: "BIOS" skip_hash_calculation: true + structure: + psx: "{category}/{gameDir}" + nes: "{gameFile}" scan: priority: diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index b1763319b7..a1468042fe 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -1,11 +1,14 @@ import os from pathlib import Path +import pytest + from config.config_manager import ( DEFAULT_EXCLUDED_DIRS, DEFAULT_EXCLUDED_EXTENSIONS, DEFAULT_EXCLUDED_FILES, ConfigManager, + parse_library_structure, ) @@ -79,6 +82,18 @@ def test_config_loader(): assert loader.config.SCAN_LANGUAGE_PRIORITY == ["jp", "es"] assert loader.config.GAMELIST_MEDIA_THUMBNAIL == "box3d" assert loader.config.GAMELIST_MEDIA_IMAGE == "title_screen" + assert loader.config.STRUCTURE_TEMPLATES == { + "psx": "{category}/{gameDir}", + "nes": "{gameFile}", + } + # The accessor parses templates on demand; unset platforms get None. + psx = loader.config.platform_structure("psx") + assert psx is not None and psx.each_file_is_game is False + assert len(psx.levels) == 1 and psx.levels[0].literal is None + nes = loader.config.platform_structure("nes") + assert nes is not None and nes.each_file_is_game is True + assert nes.levels == () + assert loader.config.platform_structure("snes") is None def test_empty_config_loader(): @@ -113,6 +128,42 @@ def test_empty_config_loader(): assert loader.config.EJS_CONTROLS == {} assert loader.config.GAMELIST_MEDIA_THUMBNAIL == "box2d" assert loader.config.GAMELIST_MEDIA_IMAGE == "screenshot" + assert loader.config.STRUCTURE_TEMPLATES == {} + + +@pytest.mark.parametrize( + ("template", "levels", "each_file_is_game"), + [ + ("{gameFile}", (), True), + ("{gameDir}", (), False), + ("{category}/{gameFile}", (None,), True), + ("Hacks/{gameFile}", ("Hacks",), True), + ("{region}/{system}/{gameDir}", (None, None), False), + ("roms/{region}/{gameFile}", ("roms", None), True), + ], +) +def test_parse_library_structure_valid(template, levels, each_file_is_game): + structure = parse_library_structure(template) + assert structure.each_file_is_game is each_file_is_game + assert tuple(level.literal for level in structure.levels) == levels + + +@pytest.mark.parametrize( + "template", + [ + "", + "justliteral", + "{category}", # no terminal + "{gameFile}/{gameDir}", # terminal not last + "{gameDir}/extra", # terminal not last + "{platform}/{gameFile}", # reserved macro RomM resolves itself + "{library}/{gameFile}", + "{}/{gameFile}", # empty macro + ], +) +def test_parse_library_structure_invalid(template): + with pytest.raises(ValueError): + parse_library_structure(template) def test_missing_config_file_is_created(tmp_path): diff --git a/backend/tests/endpoints/sockets/test_scan.py b/backend/tests/endpoints/sockets/test_scan.py index 0dfd2ee52c..9c1929993f 100644 --- a/backend/tests/endpoints/sockets/test_scan.py +++ b/backend/tests/endpoints/sockets/test_scan.py @@ -352,9 +352,9 @@ def _fs_rom(fs_name: str, fs_path: str) -> FSRom: class TestReconcileRelocatedRoms: - """A rom whose on-disk path changed (subfolder scanning) is relocated in - place by content hash instead of being re-imported as new, so its DB row — - and the saves/history/favorites/collections attached to it — survives.""" + """A rom whose on-disk path changed (custom library structure) is relocated + in place by content hash instead of being re-imported as new, so its DB row + — and the saves/history/favorites/collections attached to it — survives.""" @pytest.mark.asyncio async def test_moved_file_is_relocated_not_reimported( @@ -377,7 +377,7 @@ async def test_moved_file_is_relocated_not_reimported( ) ) - # On disk the file now lives inside a subfolder (same bytes). + # On disk the file now lives inside a nested folder (same bytes). sub = tmp_path / base / "Hacks" sub.mkdir(parents=True) (sub / "Mover.bin").write_bytes(content) diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index 536e14efd7..c0a053a59c 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -377,7 +377,7 @@ async def test_get_roms(self, handler: FSRomsHandler, platform, config): # Check excluded files are not present assert "excluded_test.tmp" not in rom_names - def _make_subfolder_config(self, subfolders: dict[str, bool | list[str]]) -> Config: + def _make_structure_config(self, templates: dict[str, str]) -> Config: return Config( EXCLUDED_PLATFORMS=[], EXCLUDED_SINGLE_EXT=["tmp"], @@ -389,176 +389,175 @@ def _make_subfolder_config(self, subfolders: dict[str, bool | list[str]]) -> Con PLATFORMS_VERSIONS={}, ROMS_FOLDER_NAME="roms", FIRMWARE_FOLDER_NAME="bios", - SCAN_SUBFOLDERS=subfolders, + STRUCTURE_TEMPLATES=templates, ) - def _build_subfolder_library(self, tmp_path: Path, platform: Platform) -> None: - """Library with a flat rom, a grouping subfolder (with a nested - sub-subfolder and a basename colliding with the root), a hidden folder, - and a folder-style multi-file rom.""" - roms = tmp_path / platform.fs_slug / "roms" + def _build_structure_library(self, tmp_path: Path, base: str) -> None: + """Library exercising literal/wildcard levels, file-vs-folder terminals, + a name collision across folders, a hidden folder, and depth > 1.""" + roms = tmp_path / base roms.mkdir(parents=True) - (roms / "Game A.zip").write_text("a") - group = roms / "All but the Best" - group.mkdir() - (group / "Game A.zip").write_text("dup") # same basename, different folder - (group / "Hidden Gem.zip").write_text("c") - deeper = group / "deeper" - deeper.mkdir() - (deeper / "Way Down.zip").write_text("d") + (roms / "Top.zip").write_text("t") + hacks = roms / "Hacks" + hacks.mkdir() + (hacks / "Shared.zip").write_text("a") + (hacks / "HackOnly.zip").write_text("b") + inner = hacks / "Inner" + inner.mkdir() + (inner / "x.bin").write_text("x") + trans = roms / "Translations" + trans.mkdir() + (trans / "Shared.zip").write_text("c") # collides with Hacks/Shared.zip hidden = roms / ".hidden" hidden.mkdir() - (hidden / "disc.chd").write_text("x") - multi = roms / "Multi Disc Game" - multi.mkdir() - (multi / "disc1.bin").write_text("p") + (hidden / "secret.zip").write_text("s") + region = roms / "Region" + region.mkdir() + usa = region / "USA" + usa.mkdir() + (usa / "usagame.zip").write_text("u") @pytest.mark.asyncio - async def test_get_roms_subfolders_disabled( + async def test_get_roms_no_structure_is_default( self, platform: Platform, tmp_path: Path ): - """By default a subfolder is a single multi-file rom, not a group.""" - self._build_subfolder_library(tmp_path, platform) + """Without a template, only top-level files and folders are surfaced + (each file a flat rom, each folder a single multi-file rom).""" handler = FSRomsHandler() handler.base_path = tmp_path - with patch( "handler.filesystem.roms_handler.cm.get_config", - lambda: self._make_subfolder_config({}), + lambda: self._make_structure_config({}), ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) roms = await handler.get_roms(platform) count = await handler.count_roms(platform) keys = {(r["fs_path"], r["fs_name"]) for r in roms} - base = f"{platform.fs_slug}/roms" - assert (base, "Game A.zip") in keys - # Directories surface as multi-file roms, not as groups. - assert (base, "All but the Best") in keys - assert (base, "Multi Disc Game") in keys + assert (base, "Top.zip") in keys + assert (base, "Hacks") in keys + assert (base, "Translations") in keys + assert (base, "Region") in keys assert (base, ".hidden") in keys - # Nothing inside any subfolder is surfaced. - assert not any(r["fs_path"] != base for r in roms) - assert len(roms) == 4 + # Nothing nested is surfaced. + assert all(r["fs_path"] == base for r in roms) + assert len(roms) == 5 assert count == len(roms) @pytest.mark.asyncio - async def test_get_roms_subfolders_enabled( + async def test_get_roms_structure_wildcard_file_terminal( self, platform: Platform, tmp_path: Path ): - """With subfolder scanning on, groups are descended into (recursively), - hidden folders are skipped, and identically-named files in different - folders stay distinct via their full path.""" - self._build_subfolder_library(tmp_path, platform) + """`{category}/{gameFile}` descends one wildcard level and treats each + file as a rom; hidden folders are skipped and identically-named files in + different folders stay distinct via their full path.""" handler = FSRomsHandler() handler.base_path = tmp_path - with patch( "handler.filesystem.roms_handler.cm.get_config", - lambda: self._make_subfolder_config({platform.fs_slug: True}), + lambda: self._make_structure_config( + {platform.fs_slug: "{category}/{gameFile}"} + ), ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) roms = await handler.get_roms(platform) count = await handler.count_roms(platform) - base = f"{platform.fs_slug}/roms" keys = {(r["fs_path"], r["fs_name"]) for r in roms} - assert (base, "Game A.zip") in keys - assert (f"{base}/All but the Best", "Game A.zip") in keys # collision kept - assert (f"{base}/All but the Best", "Hidden Gem.zip") in keys - assert (f"{base}/All but the Best/deeper", "Way Down.zip") in keys - # Group folders are descended into, so "Multi Disc Game" parts surface. - assert (f"{base}/Multi Disc Game", "disc1.bin") in keys - # Hidden folder is not descended into; it remains a multi-file rom. - assert (base, ".hidden") in keys - assert (f"{base}/.hidden", "disc.chd") not in keys - # All full-path keys are unique and count matches. + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + assert (f"{base}/Translations", "Shared.zip") in keys # collision kept + # Top-level file isn't surfaced (template requires one level down). + assert (base, "Top.zip") not in keys + # A folder at the terminal level isn't a {gameFile} rom. + assert (f"{base}/Hacks", "Inner") not in keys + # Hidden folder is never descended into. + assert not any(r["fs_path"].endswith("/.hidden") for r in roms) full_paths = [f"{r['fs_path']}/{r['fs_name']}" for r in roms] assert len(full_paths) == len(set(full_paths)) - assert sum(1 for r in roms if r["fs_name"] == "Game A.zip") == 2 + assert sum(1 for r in roms if r["fs_name"] == "Shared.zip") == 2 assert count == len(roms) @pytest.mark.asyncio - async def test_get_roms_subfolders_descriptor_dir_kept_whole( + async def test_get_roms_structure_wildcard_dir_terminal( self, platform: Platform, tmp_path: Path ): - """A subfolder holding a disc/playlist descriptor (.m3u, .cue, ...) is a - multi-disc game: it stays a single multi-file rom even with subfolder - scanning on (not split per disc), while a plain grouping folder is still - recursed.""" - roms = tmp_path / platform.fs_slug / "roms" - roms.mkdir(parents=True) - (roms / "Flat Game.zip").write_text("a") - # Multi-disc game declared by an .m3u playlist -> one rom. - md = roms / "Final Fantasy VII" - md.mkdir() - (md / "disc1.chd").write_text("1") - (md / "disc2.chd").write_text("2") - (md / "Final Fantasy VII.m3u").write_text("disc1.chd\ndisc2.chd") - # Multi-file game declared by a .cue descriptor (cue+bin) -> one rom. - cue = roms / "Some CD Game" - cue.mkdir() - (cue / "Some CD Game.cue").write_text('FILE "Some CD Game.bin" BINARY') - (cue / "Some CD Game.bin").write_text("data") - # Plain grouping folder (no descriptor) -> recursed into. - grp = roms / "Hacks" - grp.mkdir() - (grp / "Hack A.zip").write_text("h") - + """`{category}/{gameDir}` treats each folder at the terminal level as a + single multi-file rom (kept whole — no disc-splitting guesswork).""" handler = FSRomsHandler() handler.base_path = tmp_path with patch( "handler.filesystem.roms_handler.cm.get_config", - lambda: self._make_subfolder_config({platform.fs_slug: True}), + lambda: self._make_structure_config( + {platform.fs_slug: "{category}/{gameDir}"} + ), ): - roms_found = await handler.get_roms(platform) + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) count = await handler.count_roms(platform) - base = f"{platform.fs_slug}/roms" - keys = {(r["fs_path"], r["fs_name"]) for r in roms_found} - # The .m3u folder stays a single multi-file rom (not split per disc). - assert (base, "Final Fantasy VII") in keys - assert (f"{base}/Final Fantasy VII", "disc1.chd") not in keys - # The .cue+.bin folder stays whole too (the case raised in review). - assert (base, "Some CD Game") in keys - assert (f"{base}/Some CD Game", "Some CD Game.bin") not in keys - # A normal grouping folder is still recursed. - assert (f"{base}/Hacks", "Hack A.zip") in keys - assert (base, "Flat Game.zip") in keys - assert count == len(roms_found) + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + # Folders at the terminal level become whole multi-file roms... + assert (f"{base}/Hacks", "Inner") in keys + assert (f"{base}/Region", "USA") in keys + # ...and are not split into their contents. + assert (f"{base}/Hacks/Inner", "x.bin") not in keys + # Files at the terminal level are not {gameDir} roms. + assert (f"{base}/Hacks", "Shared.zip") not in keys + assert all(r["nested"] for r in roms) + assert count == len(roms) @pytest.mark.asyncio - async def test_get_roms_subfolders_named_list( + async def test_get_roms_structure_two_wildcard_levels( self, platform: Platform, tmp_path: Path ): - """A list value recurses only the named folders; every other folder - (including a folder-based multi-file game) stays a single multi-file - rom, and a non-named folder nested inside a named one is not split.""" - self._build_subfolder_library(tmp_path, platform) + """Multiple wildcard levels descend depth-first to the terminal.""" handler = FSRomsHandler() handler.base_path = tmp_path - with patch( "handler.filesystem.roms_handler.cm.get_config", - lambda: self._make_subfolder_config( - {platform.fs_slug: ["All but the Best"]} + lambda: self._make_structure_config( + {platform.fs_slug: "{region}/{system}/{gameFile}"} ), ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) roms = await handler.get_roms(platform) count = await handler.count_roms(platform) - base = f"{platform.fs_slug}/roms" keys = {(r["fs_path"], r["fs_name"]) for r in roms} - # Named folder is recursed -> its files become individual roms. - assert (f"{base}/All but the Best", "Game A.zip") in keys - assert (f"{base}/All but the Best", "Hidden Gem.zip") in keys - # A non-named folder nested inside it is kept whole (not recursed). - assert (f"{base}/All but the Best", "deeper") in keys - assert (f"{base}/All but the Best/deeper", "Way Down.zip") not in keys - # Folders NOT in the list stay as single multi-file roms. - assert (base, "Multi Disc Game") in keys - assert (f"{base}/Multi Disc Game", "disc1.bin") not in keys - # Top-level flat file and hidden folder behave as usual. - assert (base, "Game A.zip") in keys - assert (base, ".hidden") in keys + assert (f"{base}/Hacks/Inner", "x.bin") in keys + assert (f"{base}/Region/USA", "usagame.zip") in keys + # Depth-1 files are not surfaced at a depth-2 terminal. + assert (f"{base}/Hacks", "Shared.zip") not in keys + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_literal_level( + self, platform: Platform, tmp_path: Path + ): + """A literal section matches that exact folder only; sibling folders are + ignored.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config({platform.fs_slug: "Hacks/{gameFile}"}), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + # The literal only matched "Hacks"; "Translations" is ignored. + assert (f"{base}/Translations", "Shared.zip") not in keys + assert all(r["fs_path"] == f"{base}/Hacks" for r in roms) assert count == len(roms) @pytest.mark.asyncio diff --git a/backend/tests/handler/test_db_handler.py b/backend/tests/handler/test_db_handler.py index 932dc058dc..1561c8eebf 100644 --- a/backend/tests/handler/test_db_handler.py +++ b/backend/tests/handler/test_db_handler.py @@ -546,7 +546,7 @@ def test_bulk_mark_present_chunking(platform: Platform): def test_get_roms_by_fs_name_keys_on_full_path(platform: Platform): - """Identically-named files in different subfolders must stay distinct: the + """Identically-named files in different folders must stay distinct: the result is keyed on full path (fs_path/fs_name), not just the file name.""" root = db_rom_handler.add_rom( Rom( diff --git a/examples/config.example.yml b/examples/config.example.yml index e379769225..c7a7d7dabd 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -76,6 +76,44 @@ # roms_folder: 'roms' # # Skip file hash calculations on low power devices (eg. Raspberry PI) # skip_hash_calculation: false +# # Custom library structure, per platform. By default a platform's ROM folder +# # is scanned one level deep: each top-level file is a game and each top-level +# # folder is a single multi-file game. A template lets you describe a deeper / +# # organized layout instead. It is opt-in per platform (by fs_slug); platforms +# # left out keep the default behavior. +# # +# # Syntax (Retrom-style): `/`-separated path sections, relative to the +# # platform's ROM folder (RomM already resolves the library root, the +# # roms_folder above, and the platform directory — so {library}/{platform} +# # are not used here). A section wrapped in braces is a macro; a bare section +# # is a literal folder name matched exactly. The LAST section must be a +# # terminal macro: +# # - {gameFile} -> each file at that level is its own game; +# # - {gameDir} -> each folder at that level is a single multi-file game. +# # Any other braced section (e.g. {region}, {category}) is a wildcard +# # directory level: it matches any folder and is purely organizational. +# # +# # Notes: +# # - Hidden (dot-prefixed) folders are never descended into or surfaced. +# # - {gameDir} keeps a folder whole (multi-disc / cue+bin games), so there +# # is no guessing — you declare file-vs-folder explicitly. +# # - Identity is content-based on scan: moving or renaming a game within the +# # structure is detected by its hash and relocated in place, so its saves, +# # play history, favorites and collection membership follow it (as long as +# # hashing is enabled and the platform is hashable). When a folder that was +# # previously scanned as one multi-file game is now descended into, the old +# # entry is marked "missing from filesystem" (the scan log flags it); +# # delete the stale entry to clean up. +# # - Two files with the same name in different folders become distinct games; +# # gamelist.xml metadata matching is by filename, so both may match the +# # same gamelist entry. +# structure: +# # roms/nes/Hacks/foo.nes, roms/nes/Translations/bar.nes -> each file a game +# nes: '{category}/{gameFile}' +# # roms/ps3/Disc/Game/, roms/ps3/PSN/Game/ -> each folder a multi-file game +# ps3: '{category}/{gameDir}' +# # roms/snes/USA/foo.sfc, roms/snes/Japan/bar.sfc +# snes: '{region}/{gameFile}' # scan: # # Metadata priority during scans @@ -145,38 +183,6 @@ # image: screenshot # Use as the tag in the exported gamelist.xml # pegasus: # export: false # Whether to export metadata.pegasus.txt for Pegasus -# # Recurse into platform subfolders, treating nested files as their own -# # ROMs rather than collapsing each subfolder into a single multi-file ROM. -# # Opt-in per platform. The value is either: -# # - true -> recurse EVERY subfolder of the platform; -# # - a list of folder names -> recurse ONLY those folders, leaving every -# # other folder as a single multi-file ROM (so folder-based multi-file -# # games elsewhere on the platform stay intact — the recommended form -# # when a platform mixes grouping folders with multi-file games); -# # - false / omitted -> default behavior (a subfolder is one multi-file ROM). -# # -# # Behavior & notes (read before enabling): -# # - Hidden (dot-prefixed) folders are never descended into. -# # - A subfolder containing a disc/playlist descriptor (.m3u, .cue, .gdi, -# # .ccd, .toc) is kept whole as a single multi-file ROM (a multi-disc -# # game) and is NOT split into one ROM per disc, even when recursed. -# # - With `true`, a multi-file game stored as a bare folder with no such -# # descriptor WOULD be split into separate ROMs. Use the named-list form -# # (don't list that game's folder) to keep it whole. -# # - Identity is content-based on scan: moving (or renaming) a ROM between -# # subfolders is detected by its hash and relocated in place, so its -# # saves, play history, favorites and collection membership follow it — -# # as long as hashing is enabled and the platform is hashable. When a -# # folder previously scanned as one multi-file ROM is now recursed, that -# # old entry is marked "missing from filesystem" (the scan log flags it); -# # delete the stale entry to clean up. -# # - Two files with the same name in different subfolders become distinct -# # ROMs; gamelist.xml metadata matching is by filename, so both may match -# # the same gamelist entry. -# subfolders: -# nes: true # recurse every NES subfolder -# snes: ["Hacks", "Translations"] # recurse only these; keep other folders whole -# megadrive: false # EmulatorJS per-core options # emulatorjs: From f7f0ca9c2c3bc92115e8443c6ec8adbd440bc331 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 01:45:07 +0000 Subject: [PATCH 3/4] feat: allow a list of structure templates per platform (union discovery) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single fixed-depth template can't describe a platform that holds loose games in its root AND organizes others into grouping subfolders — `{gameFile}` drops the grouped games, `{category}/{gameFile}` drops the loose ones. Let a platform's `filesystem.structure` value be a list of templates; discovery is their union, deduplicated by full path: structure: nes: - "{gameFile}" # loose top-level games - "{category}/{gameFile}" # games inside grouping subfolders - config: accept str | list[str]; platform_structure() returns a tuple of parsed structures; parse_platform_structures() + validation handle both forms. - fs handler: _collect_fs_roms unions each structure's discovery, dedup by (fs_path, fs_name). - tests: list parsing, per-platform list config loading, and the mixed loose+grouped discovery case. - docs: config.example.yml leads with the mixed-layout list example. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Sk2dzM3K9qxWBdPAeGb7us --- backend/config/config_manager.py | 68 +++++++++++++------ backend/handler/filesystem/roms_handler.py | 24 +++++-- .../tests/config/fixtures/config/config.yml | 4 +- backend/tests/config/test_config_loader.py | 32 +++++++-- .../handler/filesystem/test_roms_handler.py | 34 ++++++++++ examples/config.example.yml | 13 +++- 6 files changed, 143 insertions(+), 32 deletions(-) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 1e414ba0b2..4114572b38 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -121,6 +121,26 @@ def parse_library_structure(template: str) -> LibraryStructure: return LibraryStructure(levels=tuple(levels), each_file_is_game=each_file_is_game) +def parse_platform_structures( + value: str | list[str], +) -> tuple[LibraryStructure, ...]: + """Parse a platform's custom structure config into one or more structures. + + A platform may declare a single template (string) or several (list) — the + list form lets one platform mix layouts, e.g. loose games at the root plus + games inside grouping subfolders:: + + nes: + - "{gameFile}" + - "{category}/{gameFile}" + + Discovery is the union of all listed templates. Raises ``ValueError`` if any + template is invalid. + """ + templates = [value] if isinstance(value, str) else list(value) + return tuple(parse_library_structure(template) for template in templates) + + ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config" ROMM_USER_CONFIG_FILE: Final = f"{ROMM_USER_CONFIG_PATH}/config.yml" SQLITE_DB_BASE_PATH: Final = f"{ROMM_BASE_PATH}/database" @@ -211,7 +231,7 @@ class Config: PLATFORMS_VERSIONS: dict[str, str] ROMS_FOLDER_NAME: str FIRMWARE_FOLDER_NAME: str - STRUCTURE_TEMPLATES: dict[str, str] + STRUCTURE_TEMPLATES: dict[str, str | list[str]] SKIP_HASH_CALCULATION: bool EJS_DEBUG: bool EJS_CACHE_LIMIT: int | None @@ -254,18 +274,20 @@ def has_structure_path_b(self) -> bool: return False - def platform_structure(self, fs_slug: str) -> LibraryStructure | None: - """The custom library structure for a platform, or ``None``. + def platform_structure(self, fs_slug: str) -> tuple[LibraryStructure, ...] | None: + """The custom library structure(s) for a platform, or ``None``. Opt-in per platform via `filesystem.structure` in config.yml (a map of - ``fs_slug -> template``). When unset, the platform uses RomM's default - discovery (top-level files and folders). The template is validated at - load time, so parsing here is expected to succeed. + ``fs_slug -> template`` or ``fs_slug -> [template, ...]``). When unset, + the platform uses RomM's default discovery (top-level files and + folders). Returns the parsed structures whose discovery is unioned; + templates are validated at load time, so parsing here is expected to + succeed. """ - template = getattr(self, "STRUCTURE_TEMPLATES", {}).get(fs_slug) - if not template: + value = getattr(self, "STRUCTURE_TEMPLATES", {}).get(fs_slug) + if not value: return None - return parse_library_structure(template) + return parse_platform_structures(value) class ConfigManager: @@ -775,21 +797,27 @@ def _validate_config(self): "Invalid config.yml: filesystem.structure must be a dictionary" ) sys.exit(3) - for fs_slug, template in self.config.STRUCTURE_TEMPLATES.items(): - if not isinstance(template, str): + for fs_slug, value in self.config.STRUCTURE_TEMPLATES.items(): + is_str = isinstance(value, str) + is_str_list = isinstance(value, list) and all( + isinstance(t, str) for t in value + ) + if not (is_str or is_str_list): log.critical( f"Invalid config.yml: filesystem.structure.{fs_slug} must be a " - "string template" - ) - sys.exit(3) - try: - parse_library_structure(template) - except ValueError as exc: - log.critical( - f"Invalid config.yml: filesystem.structure.{fs_slug} " - f"('{template}'): {exc}" + "template string or a list of template strings" ) sys.exit(3) + templates = [value] if is_str else value + for template in templates: + try: + parse_library_structure(template) + except ValueError as exc: + log.critical( + f"Invalid config.yml: filesystem.structure.{fs_slug} " + f"('{template}'): {exc}" + ) + sys.exit(3) # Drop unknown media types rather than exiting, since a newer release # may ship sample configs referencing media types this version doesn't know. diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index a407a6601e..0c8a22816a 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -749,12 +749,28 @@ async def _discover_structured_roms( return fs_roms async def _collect_fs_roms(self, platform: Platform) -> list[dict]: - """Discover a platform's roms, honoring its custom structure if set.""" + """Discover a platform's roms, honoring its custom structure if set. + + A platform may declare several structure templates (e.g. loose games at + the root plus games inside grouping subfolders); discovery is their + union, deduplicated by full path so overlapping templates don't surface + a rom twice. + """ rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) - structure = cm.get_config().platform_structure(platform.fs_slug) - if structure is None: + structures = cm.get_config().platform_structure(platform.fs_slug) + if structures is None: return await self._discover_default_roms(rel_roms_path) - return await self._discover_structured_roms(rel_roms_path, structure) + + fs_roms: list[dict] = [] + seen: set[tuple[str, str]] = set() + for structure in structures: + for rom in await self._discover_structured_roms(rel_roms_path, structure): + key = (rom["fs_path"], rom["fs_name"]) + if key in seen: + continue + seen.add(key) + fs_roms.append(rom) + return fs_roms async def count_roms(self, platform: Platform) -> int: """Return the number of filesystem roms for a platform without diff --git a/backend/tests/config/fixtures/config/config.yml b/backend/tests/config/fixtures/config/config.yml index 1bf6ecfb64..1448ab9549 100644 --- a/backend/tests/config/fixtures/config/config.yml +++ b/backend/tests/config/fixtures/config/config.yml @@ -32,7 +32,9 @@ filesystem: skip_hash_calculation: true structure: psx: "{category}/{gameDir}" - nes: "{gameFile}" + nes: + - "{gameFile}" + - "{category}/{gameFile}" scan: priority: diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index a1468042fe..b37efd6409 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -9,6 +9,7 @@ DEFAULT_EXCLUDED_FILES, ConfigManager, parse_library_structure, + parse_platform_structures, ) @@ -84,15 +85,19 @@ def test_config_loader(): assert loader.config.GAMELIST_MEDIA_IMAGE == "title_screen" assert loader.config.STRUCTURE_TEMPLATES == { "psx": "{category}/{gameDir}", - "nes": "{gameFile}", + "nes": ["{gameFile}", "{category}/{gameFile}"], } # The accessor parses templates on demand; unset platforms get None. psx = loader.config.platform_structure("psx") - assert psx is not None and psx.each_file_is_game is False - assert len(psx.levels) == 1 and psx.levels[0].literal is None + assert psx is not None and len(psx) == 1 + assert psx[0].each_file_is_game is False + assert len(psx[0].levels) == 1 and psx[0].levels[0].literal is None + # The list form yields one structure per template (union on discovery). nes = loader.config.platform_structure("nes") - assert nes is not None and nes.each_file_is_game is True - assert nes.levels == () + assert nes is not None and len(nes) == 2 + assert nes[0].each_file_is_game is True and nes[0].levels == () + assert nes[1].each_file_is_game is True + assert len(nes[1].levels) == 1 and nes[1].levels[0].literal is None assert loader.config.platform_structure("snes") is None @@ -166,6 +171,23 @@ def test_parse_library_structure_invalid(template): parse_library_structure(template) +def test_parse_platform_structures_string_and_list(): + # A bare string yields a single structure. + single = parse_platform_structures("{gameFile}") + assert len(single) == 1 and single[0].each_file_is_game is True + + # A list yields one structure per template, preserving order. + multi = parse_platform_structures(["{gameFile}", "{category}/{gameDir}"]) + assert len(multi) == 2 + assert multi[0].levels == () and multi[0].each_file_is_game is True + assert len(multi[1].levels) == 1 and multi[1].each_file_is_game is False + + +def test_parse_platform_structures_propagates_invalid(): + with pytest.raises(ValueError): + parse_platform_structures(["{gameFile}", "nope"]) + + def test_missing_config_file_is_created(tmp_path): config_file = tmp_path / "config" / "config.yml" diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index c0a053a59c..509f80fc80 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -560,6 +560,40 @@ async def test_get_roms_structure_literal_level( assert all(r["fs_path"] == f"{base}/Hacks" for r in roms) assert count == len(roms) + @pytest.mark.asyncio + async def test_get_roms_structure_list_unions_loose_and_grouped( + self, platform: Platform, tmp_path: Path + ): + """A list of templates unions their discovery: loose top-level games + (`{gameFile}`) plus games inside grouping subfolders + (`{category}/{gameFile}`) — the common mixed layout — without dropping + either, deduplicated by full path.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config( + {platform.fs_slug: ["{gameFile}", "{category}/{gameFile}"]} + ), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + # Loose top-level game ({gameFile}). + assert (base, "Top.zip") in keys + # Grouped games one level down ({category}/{gameFile}). + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + assert (f"{base}/Translations", "Shared.zip") in keys + # Hidden folder still skipped; no duplicate full paths. + assert not any(r["fs_path"].endswith("/.hidden") for r in roms) + full_paths = [f"{r['fs_path']}/{r['fs_name']}" for r in roms] + assert len(full_paths) == len(set(full_paths)) + assert count == len(roms) + @pytest.mark.asyncio async def test_get_rom_files_single_rom( self, handler: FSRomsHandler, rom_single, config diff --git a/examples/config.example.yml b/examples/config.example.yml index c7a7d7dabd..ee268c9e49 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -93,6 +93,11 @@ # # Any other braced section (e.g. {region}, {category}) is a wildcard # # directory level: it matches any folder and is purely organizational. # # +# # A platform may also declare a LIST of templates instead of one; discovery +# # is their union. This covers the common mixed layout — loose games at the +# # platform root AND games inside grouping subfolders below it — which a +# # single fixed-depth template can't express on its own. +# # # # Notes: # # - Hidden (dot-prefixed) folders are never descended into or surfaced. # # - {gameDir} keeps a folder whole (multi-disc / cue+bin games), so there @@ -108,8 +113,12 @@ # # gamelist.xml metadata matching is by filename, so both may match the # # same gamelist entry. # structure: -# # roms/nes/Hacks/foo.nes, roms/nes/Translations/bar.nes -> each file a game -# nes: '{category}/{gameFile}' +# # Mixed: loose top-level games AND grouping subfolders below them, e.g. +# # roms/nes/game01.nes (matched by {gameFile}) +# # roms/nes/Hacks/game03.nes (matched by {category}/{gameFile}) +# nes: +# - '{gameFile}' +# - '{category}/{gameFile}' # # roms/ps3/Disc/Game/, roms/ps3/PSN/Game/ -> each folder a multi-file game # ps3: '{category}/{gameDir}' # # roms/snes/USA/foo.sfc, roms/snes/Japan/bar.sfc From 4df54cee7202d608d021f2fa9353510865ad970f Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Tue, 23 Jun 2026 22:47:48 -0400 Subject: [PATCH 4/4] run fmt --- backend/config/config_manager.py | 1 - .../handler/filesystem/test_roms_handler.py | 54 +++++++++---------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 27d41d836e..9b40db883e 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -29,7 +29,6 @@ from logger.formatter import highlight as hl from logger.logger import log - # Terminal macros for a custom library structure template. Exactly one must # appear, as the last path segment. STRUCTURE_GAME_FILE: Final = "gameFile" diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index 509f80fc80..cc8b19d5ab 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -955,19 +955,19 @@ async def test_top_level_files_only_in_main_hash( # The main ROM hash should be different from the translation file hash # (this verifies that the translation is not included in the main hash) - assert parsed_rom_files.md5_hash == base_game_rom_file.md5_hash, ( - "Main ROM hash should include base game file" - ) - assert parsed_rom_files.md5_hash != translation_rom_file.md5_hash, ( - "Main ROM hash should not include translation file" - ) + assert ( + parsed_rom_files.md5_hash == base_game_rom_file.md5_hash + ), "Main ROM hash should include base game file" + assert ( + parsed_rom_files.md5_hash != translation_rom_file.md5_hash + ), "Main ROM hash should not include translation file" - assert parsed_rom_files.sha1_hash == base_game_rom_file.sha1_hash, ( - "Main ROM hash should include base game file" - ) - assert parsed_rom_files.sha1_hash != translation_rom_file.sha1_hash, ( - "Main ROM hash should not include translation file" - ) + assert ( + parsed_rom_files.sha1_hash == base_game_rom_file.sha1_hash + ), "Main ROM hash should include base game file" + assert ( + parsed_rom_files.sha1_hash != translation_rom_file.sha1_hash + ), "Main ROM hash should not include translation file" @pytest.mark.asyncio async def test_get_rom_files_with_chd_v5_uses_internal_hash( @@ -1014,9 +1014,9 @@ async def test_get_rom_files_with_chd_v5_uses_internal_hash( assert len(parsed_rom_files.rom_files) == 1 assert parsed_rom_files.crc_hash != "", "CRC should be computed from raw bytes" assert parsed_rom_files.md5_hash != "", "MD5 should be computed from raw bytes" - assert parsed_rom_files.sha1_hash != "", ( - "SHA1 should be computed from raw bytes" - ) + assert ( + parsed_rom_files.sha1_hash != "" + ), "SHA1 should be computed from raw bytes" # Raw file SHA1 is NOT the header SHA1 assert parsed_rom_files.sha1_hash != internal_sha1 @@ -1294,15 +1294,15 @@ async def test_get_rom_files_with_non_v5_chd_fallback_to_std_hashing( # All hashes should be populated (calculated from file content) assert len(parsed_rom_files.rom_files) == 1 - assert parsed_rom_files.crc_hash != "", ( - "CRC hash should be calculated for non-v5 CHD" - ) - assert parsed_rom_files.md5_hash != "", ( - "MD5 hash should be calculated for non-v5 CHD" - ) - assert parsed_rom_files.sha1_hash != "", ( - "SHA1 hash should be calculated for non-v5 CHD" - ) + assert ( + parsed_rom_files.crc_hash != "" + ), "CRC hash should be calculated for non-v5 CHD" + assert ( + parsed_rom_files.md5_hash != "" + ), "MD5 hash should be calculated for non-v5 CHD" + assert ( + parsed_rom_files.sha1_hash != "" + ), "SHA1 hash should be calculated for non-v5 CHD" # Verify they're actual hash values (not from an internal header) assert parsed_rom_files.rom_files[0].crc_hash == parsed_rom_files.crc_hash @@ -1715,9 +1715,9 @@ def test_extract_chd_hash_off_by_one_header_sizes(self, tmp_path): result = extract_chd_hash(chd_file) - assert result == expected, ( - f"Failed for size {size}: got {result}, expected {expected}" - ) + assert ( + result == expected + ), f"Failed for size {size}: got {result}, expected {expected}" def test_extract_chd_hash_corrupted_header_data(self, tmp_path): """Test handling of corrupted/invalid data in header fields"""