diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 80ea2068ca..9b40db883e 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -4,6 +4,7 @@ import json import os import sys +from dataclasses import dataclass from pathlib import Path from typing import Final, NotRequired, TypedDict @@ -28,6 +29,117 @@ from logger.formatter import highlight as hl from logger.logger import log +# Terminal macros for a custom library structure template. Exactly one must +# appear, as the last path segment. +STRUCTURE_GAME_FILE: Final = "gameFile" +STRUCTURE_GAME_DIR: Final = "gameDir" +# Macros that belong to Retrom's full-path templates but are resolved by RomM +# itself (library base + roms folder + platform discovery). A per-platform +# template is relative to the platform's ROM folder, so these are rejected with +# a helpful message instead of being silently treated as wildcard levels. +_STRUCTURE_RESERVED_NONTERMINAL: Final = frozenset({"platform", "library"}) + + +@dataclass(frozen=True) +class StructureLevel: + """One intermediate directory level of a custom structure template. + + ``literal`` is the exact folder name to match, or ``None`` for a wildcard + macro level (``{region}``, ``{category}``, …) that matches any folder. + """ + + literal: str | None + + +@dataclass(frozen=True) +class LibraryStructure: + """A parsed per-platform custom library structure. + + Relative to the platform's ROM folder: ``levels`` are the intermediate + directory levels to descend (literal or wildcard), and ``each_file_is_game`` + selects the terminal — ``{gameFile}`` (each file is a ROM) vs ``{gameDir}`` + (each directory is a multi-file ROM). + """ + + levels: tuple[StructureLevel, ...] + each_file_is_game: bool + + +def parse_library_structure(template: str) -> LibraryStructure: + """Parse a custom structure template into a ``LibraryStructure``. + + Template syntax mirrors Retrom: ``/``-separated path sections where a + section wrapped in braces is a macro and a bare section is a literal folder + name. The last section must be the terminal ``{gameFile}`` or ``{gameDir}``; + every other braced section is a wildcard directory level. The template is + relative to the platform's ROM folder, so ``{platform}`` / ``{library}`` are + not used here. + + Raises ``ValueError`` on an invalid template. + """ + sections = [s for s in template.split("/") if s != ""] + if not sections: + raise ValueError("template is empty") + + levels: list[StructureLevel] = [] + each_file_is_game: bool | None = None + + for index, section in enumerate(sections): + is_last = index == len(sections) - 1 + is_macro = section.startswith("{") and section.endswith("}") + + if not is_macro: + if "{" in section or "}" in section: + raise ValueError(f"malformed macro in section '{section}'") + levels.append(StructureLevel(literal=section)) + continue + + name = section[1:-1].strip() + if name in (STRUCTURE_GAME_FILE, STRUCTURE_GAME_DIR): + if not is_last: + raise ValueError( + f"'{{{name}}}' must be the last path section of the template" + ) + each_file_is_game = name == STRUCTURE_GAME_FILE + continue + + if name in _STRUCTURE_RESERVED_NONTERMINAL: + raise ValueError( + f"'{{{name}}}' is not supported: a platform template is relative " + "to that platform's ROM folder (RomM resolves library root, roms " + "folder and platform on its own)" + ) + if not name: + raise ValueError("empty macro '{}'") + # Any other braced section is an organizational wildcard directory level. + levels.append(StructureLevel(literal=None)) + + if each_file_is_game is None: + raise ValueError("template must end with '{gameFile}' or '{gameDir}'") + + return LibraryStructure(levels=tuple(levels), each_file_is_game=each_file_is_game) + + +def parse_platform_structures( + value: str | list[str], +) -> tuple[LibraryStructure, ...]: + """Parse a platform's custom structure config into one or more structures. + + A platform may declare a single template (string) or several (list) — the + list form lets one platform mix layouts, e.g. loose games at the root plus + games inside grouping subfolders:: + + nes: + - "{gameFile}" + - "{category}/{gameFile}" + + Discovery is the union of all listed templates. Raises ``ValueError`` if any + template is invalid. + """ + templates = [value] if isinstance(value, str) else list(value) + return tuple(parse_library_structure(template) for template in templates) + + ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config" ROMM_USER_CONFIG_FILE: Final = f"{ROMM_USER_CONFIG_PATH}/config.yml" SQLITE_DB_BASE_PATH: Final = f"{ROMM_BASE_PATH}/database" @@ -119,6 +231,7 @@ class Config: PLATFORMS_VERSIONS: dict[str, str] ROMS_FOLDER_NAME: str FIRMWARE_FOLDER_NAME: str + STRUCTURE_TEMPLATES: dict[str, str | list[str]] SKIP_HASH_CALCULATION: bool EJS_DEBUG: bool EJS_CACHE_LIMIT: int | None @@ -161,6 +274,21 @@ def has_structure_path_b(self) -> bool: return False + def platform_structure(self, fs_slug: str) -> tuple[LibraryStructure, ...] | None: + """The custom library structure(s) for a platform, or ``None``. + + Opt-in per platform via `filesystem.structure` in config.yml (a map of + ``fs_slug -> template`` or ``fs_slug -> [template, ...]``). When unset, + the platform uses RomM's default discovery (top-level files and + folders). Returns the parsed structures whose discovery is unioned; + templates are validated at load time, so parsing here is expected to + succeed. + """ + value = getattr(self, "STRUCTURE_TEMPLATES", {}).get(fs_slug) + if not value: + return None + return parse_platform_structures(value) + class ConfigManager: """ @@ -447,6 +575,9 @@ def _parse_config(self): PEGASUS_AUTO_EXPORT_ON_SCAN=pydash.get( self._raw_config, "scan.pegasus.export", False ), + STRUCTURE_TEMPLATES=pydash.get( + self._raw_config, "filesystem.structure", {} + ), ) def _get_ejs_controls(self) -> dict[str, EjsControls]: @@ -661,6 +792,33 @@ def _validate_config(self): log.critical("Invalid config.yml: scan.media must be a list") sys.exit(3) + if not isinstance(self.config.STRUCTURE_TEMPLATES, dict): + log.critical( + "Invalid config.yml: filesystem.structure must be a dictionary" + ) + sys.exit(3) + for fs_slug, value in self.config.STRUCTURE_TEMPLATES.items(): + is_str = isinstance(value, str) + is_str_list = isinstance(value, list) and all( + isinstance(t, str) for t in value + ) + if not (is_str or is_str_list): + log.critical( + f"Invalid config.yml: filesystem.structure.{fs_slug} must be a " + "template string or a list of template strings" + ) + sys.exit(3) + templates = [value] if is_str else value + for template in templates: + try: + parse_library_structure(template) + except ValueError as exc: + log.critical( + f"Invalid config.yml: filesystem.structure.{fs_slug} " + f"('{template}'): {exc}" + ) + sys.exit(3) + # Drop unknown media types rather than exiting, since a newer release # may ship sample configs referencing media types this version doesn't know. unknown_media = [ @@ -754,6 +912,7 @@ def _update_config_file(self) -> None: "filesystem": { "roms_folder": self.config.ROMS_FOLDER_NAME, "firmware_folder": self.config.FIRMWARE_FOLDER_NAME, + "structure": self.config.STRUCTURE_TEMPLATES, "skip_hash_calculation": self.config.SKIP_HASH_CALCULATION, }, "system": { diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index f9f8e92c3a..08cb4ac243 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import os +from collections import defaultdict from dataclasses import dataclass from itertools import batched from typing import Any, Final @@ -30,7 +32,7 @@ fs_resource_handler, fs_rom_handler, ) -from handler.filesystem.roms_handler import FSRom +from handler.filesystem.roms_handler import FSRom, ParsedRomFiles from handler.metadata import meta_gamelist_handler, meta_hltb_handler from handler.metadata.ss_handler import add_ss_auth_to_url, get_preferred_media_types from handler.redis_handler import get_job_func_name, high_prio_queue, redis_client @@ -247,7 +249,9 @@ async def _identify_rom( # Update properties that don't require metadata parsed_tags = fs_rom_handler.parse_tags(fs_rom["fs_name"]) - roms_path = fs_rom_handler.get_roms_fs_structure(platform.fs_slug) + # The discovered path is the rom's actual directory, which may be nested + # under the platform roms folder when a custom structure is configured. + roms_path = fs_rom["fs_path"] # Create the entry early so we have the ID newly_added: bool = rom is None @@ -491,6 +495,159 @@ async def _identify_rom( ) +def _fs_rom_size(fs_rom: FSRom) -> int: + """On-disk size of a discovered rom (a file's size, or the sum of a + folder's files). Cheap stat-only pre-filter used before hashing during + relocation matching. Returns -1 when the path can't be read. + """ + abs_path = os.path.join( + fs_rom_handler.base_path, fs_rom["fs_path"], fs_rom["fs_name"] + ) + try: + if os.path.isdir(abs_path): + total = 0 + for root, _dirs, files in os.walk(abs_path): + for name in files: + try: + total += os.stat(os.path.join(root, name)).st_size + except OSError: + pass + return total + return os.stat(abs_path).st_size + except OSError: + return -1 + + +def _hashes_match(rom: Rom, parsed: ParsedRomFiles) -> bool: + """Whether an existing rom and freshly parsed files are the same content. + + A single matching non-empty hash is enough — a collision across sha1/md5/ + crc/ra is negligible, and different platforms populate different hashes. + """ + for stored, computed in ( + (rom.sha1_hash, parsed.sha1_hash), + (rom.md5_hash, parsed.md5_hash), + (rom.crc_hash, parsed.crc_hash), + (rom.ra_hash, parsed.ra_hash), + ): + if stored and computed and stored == computed: + return True + return False + + +async def _reconcile_relocated_roms( + platform: Platform, + fs_roms: list[FSRom], +) -> set[str]: + """Relocate roms whose on-disk path changed instead of re-importing them. + + Under a custom library structure, moving (or renaming) a file reads as a + new path. Rather than insert a fresh rom and mark the old one missing — + which would drop saves, play history, favorites and collection membership — + match a newly-seen file to a now-missing rom by content hash and update that + rom's path in place. Returns the set of full paths that were fully handled + this way (so the caller skips them in the normal scan loop). + + Falls back to no-op (path-based identity) when hashes are unavailable + (``skip_hash_calculation`` or a non-hashable platform). + """ + existing = db_rom_handler.get_roms_for_relocation(platform.id) + existing_paths = {rom.full_path for rom in existing} + + present_paths = {f"{fr['fs_path']}/{fr['fs_name']}" for fr in fs_roms} + # A rom that vanished from its stored path is a relocation candidate. + disappeared = [rom for rom in existing if rom.full_path not in present_paths] + if not disappeared: + return set() + + # Group candidates by total size: a moved file keeps its size, so this is a + # cheap pre-filter that avoids hashing every newly-seen file (e.g. on first + # enable, where the whole structure reads as new). + by_size: dict[int, list[Rom]] = defaultdict(list) + for rom in disappeared: + by_size[rom.fs_size_bytes].append(rom) + + handled: set[str] = set() + claimed: set[int] = set() + calculate_hashes = not cm.get_config().SKIP_HASH_CALCULATION + + for fs_rom in fs_roms: + full_path = f"{fs_rom['fs_path']}/{fs_rom['fs_name']}" + if full_path in existing_paths: + continue # already matched by path; not a relocation + + candidates = [ + rom + for rom in by_size.get(_fs_rom_size(fs_rom), []) + if rom.id not in claimed + ] + if not candidates: + continue + + # Compute the file's identity hashes (and its rom files for the new + # location) once; reuse them for both matching and the relocation write. + transient = Rom( + fs_name=fs_rom["fs_name"], + fs_path=fs_rom["fs_path"], + platform_id=platform.id, + ) + transient.platform = platform + parsed = await fs_rom_handler.get_rom_files( + transient, calculate_hashes=calculate_hashes + ) + if not ( + parsed.sha1_hash or parsed.md5_hash or parsed.crc_hash or parsed.ra_hash + ): + continue # no usable hash to match on + + match = next((rom for rom in candidates if _hashes_match(rom, parsed)), None) + if match is None: + continue + + claimed.add(match.id) + handled.add(full_path) + + old_path = match.full_path + db_rom_handler.update_rom( + match.id, + { + "fs_name": fs_rom["fs_name"], + "fs_path": fs_rom["fs_path"], + "fs_size_bytes": sum(f.file_size_bytes for f in parsed.rom_files), + "crc_hash": parsed.crc_hash, + "md5_hash": parsed.md5_hash, + "sha1_hash": parsed.sha1_hash, + "ra_hash": parsed.ra_hash, + "missing_from_fs": False, + }, + ) + db_rom_handler.purge_rom_files(match.id) + for file in parsed.rom_files: + db_rom_handler.add_rom_file( + RomFile( + rom_id=match.id, + file_name=file.file_name, + file_path=file.file_path, + file_size_bytes=file.file_size_bytes, + last_modified=file.last_modified, + category=file.category, + audio_meta=file.audio_meta, + crc_hash=file.crc_hash, + md5_hash=file.md5_hash, + sha1_hash=file.sha1_hash, + ra_hash=file.ra_hash, + chd_sha1_hash=file.chd_sha1_hash, + ) + ) + + log.info( + f"{hl('Relocated', color=BLUE)} {hl(old_path)} → {hl(full_path)} " + "(moved on disk; metadata and user data preserved)" + ) + + return handled + + async def _identify_platform( platform_slug: str, scan_type: ScanType, @@ -582,6 +739,24 @@ async def _identify_platform( else: log.info(f"{hl(str(len(fs_roms)))} roms found in the file system") + # Detect roms that simply moved on disk (custom library structure) and + # relocate them in place so their saves/history/favorites/collections + # follow, instead of re-importing them as new and orphaning the old entry. + # Only runs for platforms with a custom structure, so default libraries pay + # no extra cost. + if cm.get_config().platform_structure(platform.fs_slug) is not None: + relocated_paths = await _reconcile_relocated_roms(platform, fs_roms) + if relocated_paths: + await scan_stats.increment( + socket_manager=socket_manager, + scanned_roms=len(relocated_paths), + ) + fs_roms = [ + fs_rom + for fs_rom in fs_roms + if f"{fs_rom['fs_path']}/{fs_rom['fs_name']}" not in relocated_paths + ] + # Create semaphore to limit concurrent ROM scanning scan_semaphore = asyncio.Semaphore(SCAN_WORKERS) @@ -602,7 +777,10 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: ) for fs_roms_batch in batched(fs_roms, 200, strict=False): - roms_by_fs_name = db_rom_handler.get_roms_by_fs_name( + # Key matches on the rom's full path (fs_path/fs_name), not just the + # file name, so identically-named files in different folders don't + # collide under a custom library structure. + roms_by_full_path = db_rom_handler.get_roms_by_fs_name( platform_id=platform.id, fs_names={fs_rom["fs_name"] for fs_rom in fs_roms_batch}, ) @@ -612,7 +790,7 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: roms_to_scan: list[tuple[FSRom, Rom | None]] = [] for fs_rom in fs_roms_batch: - rom = roms_by_fs_name.get(fs_rom["fs_name"]) + rom = roms_by_full_path.get(f"{fs_rom['fs_path']}/{fs_rom['fs_name']}") if _should_scan_rom( scan_type=scan_type, rom=rom, @@ -644,12 +822,27 @@ async def scan_rom_with_semaphore(fs_rom: FSRom, rom: Rom | None) -> None: log.error(f"Error scanning ROM {fs_rom['fs_name']}: {result}") missing_roms = db_rom_handler.mark_missing_roms( - platform.id, [rom["fs_name"] for rom in fs_roms] + platform.id, [f"{rom['fs_path']}/{rom['fs_name']}" for rom in fs_roms] ) if len(missing_roms) > 0: log.warning(f"{hl('Missing')} roms from filesystem:") + # A folder a custom structure now descends into used to be a single + # multi-file rom; that old entry shows up here as missing. Flag those so + # it's clear the "missing" is expected and the stale entry can be + # deleted. A superseded folder's path is a parent of a discovered rom. + descended_paths = {rom["fs_path"] for rom in fs_roms} for r in missing_roms: - log.warning(f" - {r.fs_name}") + superseded = any( + p == r.full_path or p.startswith(f"{r.full_path}/") + for p in descended_paths + ) + if superseded: + log.warning( + f" - {r.fs_name} (now scanned as a folder of roms — " + "delete this stale entry to clean up)" + ) + else: + log.warning(f" - {r.fs_name}") missing_firmware = db_firmware_handler.mark_missing_firmware( platform.id, [fw for fw in fs_firmware] diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index 479ebb9f35..9a2fe24e0c 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -1171,7 +1171,11 @@ def get_roms_by_fs_name( fs_names: Iterable[str], session: Session = None, # type: ignore ) -> dict[str, Rom]: - """Retrieve a dictionary of roms by their filesystem names. + """Retrieve a dictionary of roms keyed by their full path (fs_path/fs_name). + + Filters by file name for an indexed lookup, but keys the result on the + full path so identically-named files in different folders (custom + library structures) remain distinct. Eager-loads only `platform` (used downstream by the scan loop via `rom.platform_slug` / `rom.platform.fs_slug`). This deliberately @@ -1195,7 +1199,7 @@ def get_roms_by_fs_name( .all() ) - return {rom.fs_name: rom for rom in roms} + return {rom.full_path: rom for rom in roms} @begin_session def update_rom( @@ -1278,19 +1282,23 @@ def mark_missing_roms( ) -> Sequence[Rom]: """Sync `missing_from_fs` for a platform against the keep-list. + The keep-list holds rom full paths (fs_path/fs_name) so that + identically-named files in different folders are tracked + independently under a custom library structure. + Reads the rows once and writes only those whose state actually changes, so a re-scan of an unchanged platform issues no updates. """ keep_set = set(fs_roms_to_keep) rows = session.execute( - select(Rom.id, Rom.fs_name, Rom.missing_from_fs).where( + select(Rom.id, Rom.fs_path, Rom.fs_name, Rom.missing_from_fs).where( Rom.platform_id == platform_id ) ).all() flips: dict[bool, list[int]] = {True: [], False: []} - for rom_id, fs_name, was_missing in rows: - is_missing = fs_name not in keep_set + for rom_id, fs_path, fs_name, was_missing in rows: + is_missing = f"{fs_path}/{fs_name}" not in keep_set if is_missing != was_missing: flips[is_missing].append(rom_id) @@ -1305,8 +1313,12 @@ def mark_missing_roms( return ( session.scalars( + # `fs_path` is eager-loaded alongside `fs_name` so callers can + # read `rom.full_path` after the session closes (the returned + # instances are detached); without it that lazy-loads and raises + # DetachedInstanceError. select(Rom) - .options(load_only(Rom.id, Rom.fs_name)) + .options(load_only(Rom.id, Rom.fs_name, Rom.fs_path)) .where( and_( Rom.platform_id == platform_id, @@ -1319,6 +1331,40 @@ def mark_missing_roms( .all() ) + @begin_session + def get_roms_for_relocation( + self, + platform_id: int, + session: Session = None, # type: ignore + ) -> Sequence[Rom]: + """Light-weight load of a platform's roms for relocation matching. + + Returns detached rom instances carrying only the columns needed to + detect a moved/renamed file by content (identity hashes + size) and to + resolve its current `full_path`. Used by the scan loop to relocate a + rom whose on-disk path changed instead of re-importing it as new. + """ + return ( + session.scalars( + select(Rom) + .options( + load_only( + Rom.id, + Rom.fs_name, + Rom.fs_path, + Rom.fs_size_bytes, + Rom.crc_hash, + Rom.md5_hash, + Rom.sha1_hash, + Rom.ra_hash, + ) + ) + .where(Rom.platform_id == platform_id) + ) + .unique() + .all() + ) + @begin_session def add_rom_user( self, diff --git a/backend/handler/filesystem/roms_handler.py b/backend/handler/filesystem/roms_handler.py index d1a319b52c..0c8a22816a 100644 --- a/backend/handler/filesystem/roms_handler.py +++ b/backend/handler/filesystem/roms_handler.py @@ -15,6 +15,7 @@ from config.config_manager import ( DEFAULT_EXCLUDED_EXTENSIONS, DEFAULT_EXCLUDED_FILES, + LibraryStructure, ) from config.config_manager import config_manager as cm from exceptions.fs_exceptions import ( @@ -88,6 +89,7 @@ class FSRom(TypedDict): fs_name: str + fs_path: str flat: bool nested: bool files: list[RomFile] @@ -316,10 +318,11 @@ async def get_rom_files( from adapters.services.rahasher import RAHasherService from handler.metadata import meta_ra_handler - rel_roms_path = self.get_roms_fs_structure( - rom.platform.fs_slug - ) # Relative path to roms - abs_fs_path = self.validate_path(rel_roms_path) # Absolute path to roms + # The rom's stored directory is the source of truth for its location, so + # roms inside a nested folder (custom library structure) resolve to their + # real path rather than the platform roms root. + rel_roms_path = rom.fs_path # Relative path to the rom's directory + abs_fs_path = self.validate_path(rel_roms_path) # Absolute path to that dir rom_files: list[RomFile] = [] # Skip hashing games for platforms that don't have a hash database or when hashes are disabled @@ -392,14 +395,19 @@ def _largest_chd_file() -> Path | None: try: if is_top_level: # Include this file in the main ROM hash calculation - crc_c, rom_crc_c, md5_h, rom_md5_h, sha1_h, rom_sha1_h = ( - await asyncio.to_thread( - self._calculate_rom_hashes, - Path(f_path, file_name), - rom_crc_c, - rom_md5_h, - rom_sha1_h, - ) + ( + crc_c, + rom_crc_c, + md5_h, + rom_md5_h, + sha1_h, + rom_sha1_h, + ) = await asyncio.to_thread( + self._calculate_rom_hashes, + Path(f_path, file_name), + rom_crc_c, + rom_md5_h, + rom_sha1_h, ) else: # Calculate individual file hash only @@ -529,14 +537,19 @@ def _hash_raw_archive(crc: int) -> int: ) elif hashable_platform: try: - crc_c, rom_crc_c, md5_h, rom_md5_h, sha1_h, rom_sha1_h = ( - await asyncio.to_thread( - self._calculate_rom_hashes, - Path(abs_fs_path, rom.fs_name), - rom_crc_c, - rom_md5_h, - rom_sha1_h, - ) + ( + crc_c, + rom_crc_c, + md5_h, + rom_md5_h, + sha1_h, + rom_sha1_h, + ) = await asyncio.to_thread( + self._calculate_rom_hashes, + Path(abs_fs_path, rom.fs_name), + rom_crc_c, + rom_md5_h, + rom_sha1_h, ) except zlib.error: crc_c = 0 @@ -664,21 +677,110 @@ def update_hashes(chunk: bytes | bytearray): rom_sha1_h, ) + async def _discover_default_roms(self, rel_roms_path: str) -> list[dict]: + """Default discovery: top-level files and folders of the roms path. + + Each top-level file is a flat rom; each top-level directory is a single + multi-file rom. This is RomM's behavior for platforms without a custom + structure template. + """ + fs_roms: list[dict] = [ + {"fs_name": rom, "fs_path": rel_roms_path, "flat": True, "nested": False} + for rom in self.exclude_single_files(await self.list_files(rel_roms_path)) + ] + fs_roms += [ + {"fs_name": rom, "fs_path": rel_roms_path, "flat": False, "nested": True} + for rom in self.exclude_multi_roms( + await self.list_directories(rel_roms_path) + ) + ] + return fs_roms + + async def _discover_structured_roms( + self, rel_roms_path: str, structure: LibraryStructure + ) -> list[dict]: + """Discover roms following a custom library structure template. + + Descends the template's intermediate directory levels (literal names + matched exactly, wildcard macros matching any folder), then collects + roms at the terminal: ``{gameFile}`` makes each file a rom, ``{gameDir}`` + makes each directory a (multi-file) rom. Each rom records its real + ``fs_path``. Hidden (dot-prefixed) folders are never descended into or + surfaced. + """ + dirs = [rel_roms_path] + for level in structure.levels: + next_dirs: list[str] = [] + for directory in dirs: + for sub in await self.list_directories(directory): + if sub.startswith("."): + continue + if level.literal is not None and sub != level.literal: + continue + next_dirs.append(f"{directory}/{sub}") + dirs = next_dirs + + fs_roms: list[dict] = [] + for directory in dirs: + if structure.each_file_is_game: + for name in self.exclude_single_files(await self.list_files(directory)): + fs_roms.append( + { + "fs_name": name, + "fs_path": directory, + "flat": True, + "nested": False, + } + ) + else: + for name in self.exclude_multi_roms( + await self.list_directories(directory) + ): + if name.startswith("."): + continue + fs_roms.append( + { + "fs_name": name, + "fs_path": directory, + "flat": False, + "nested": True, + } + ) + return fs_roms + + async def _collect_fs_roms(self, platform: Platform) -> list[dict]: + """Discover a platform's roms, honoring its custom structure if set. + + A platform may declare several structure templates (e.g. loose games at + the root plus games inside grouping subfolders); discovery is their + union, deduplicated by full path so overlapping templates don't surface + a rom twice. + """ + rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) + structures = cm.get_config().platform_structure(platform.fs_slug) + if structures is None: + return await self._discover_default_roms(rel_roms_path) + + fs_roms: list[dict] = [] + seen: set[tuple[str, str]] = set() + for structure in structures: + for rom in await self._discover_structured_roms(rel_roms_path, structure): + key = (rom["fs_path"], rom["fs_name"]) + if key in seen: + continue + seen.add(key) + fs_roms.append(rom) + return fs_roms + async def count_roms(self, platform: Platform) -> int: """Return the number of filesystem roms for a platform without materializing FSRom objects. """ try: - rel_roms_path = self.get_roms_fs_structure(platform.fs_slug) - fs_single_roms = await self.list_files(path=rel_roms_path) - fs_multi_roms = await self.list_directories(path=rel_roms_path) + return len(await self._collect_fs_roms(platform)) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e - return len(self.exclude_single_files(fs_single_roms)) + len( - self.exclude_multi_roms(fs_multi_roms) - ) - async def get_roms(self, platform: Platform) -> list[FSRom]: """Gets all filesystem roms for a platform @@ -688,27 +790,15 @@ async def get_roms(self, platform: Platform) -> list[FSRom]: list with all the filesystem roms for a platform """ try: - rel_roms_path = self.get_roms_fs_structure( - platform.fs_slug - ) # Relative path to roms - - fs_single_roms = await self.list_files(path=rel_roms_path) - fs_multi_roms = await self.list_directories(path=rel_roms_path) + fs_roms = await self._collect_fs_roms(platform) except FileNotFoundError as e: raise RomsNotFoundException(platform=platform.fs_slug) from e - fs_roms: list[dict] = [ - {"fs_name": rom, "flat": True, "nested": False} - for rom in self.exclude_single_files(fs_single_roms) - ] + [ - {"fs_name": rom, "flat": False, "nested": True} - for rom in self.exclude_multi_roms(fs_multi_roms) - ] - return sorted( [ FSRom( fs_name=rom["fs_name"], + fs_path=rom["fs_path"], flat=rom["flat"], nested=rom["nested"], files=[], @@ -719,7 +809,7 @@ async def get_roms(self, platform: Platform) -> list[FSRom]: ) for rom in fs_roms ], - key=lambda rom: rom["fs_name"], + key=lambda rom: (rom["fs_path"], rom["fs_name"]), ) async def rename_fs_rom(self, old_name: str, new_name: str, fs_path: str) -> None: diff --git a/backend/tests/config/fixtures/config/config.yml b/backend/tests/config/fixtures/config/config.yml index 1e7c22a0db..1448ab9549 100644 --- a/backend/tests/config/fixtures/config/config.yml +++ b/backend/tests/config/fixtures/config/config.yml @@ -30,6 +30,11 @@ filesystem: roms_folder: "ROMS" firmware_folder: "BIOS" skip_hash_calculation: true + structure: + psx: "{category}/{gameDir}" + nes: + - "{gameFile}" + - "{category}/{gameFile}" scan: priority: diff --git a/backend/tests/config/test_config_loader.py b/backend/tests/config/test_config_loader.py index b1763319b7..b37efd6409 100644 --- a/backend/tests/config/test_config_loader.py +++ b/backend/tests/config/test_config_loader.py @@ -1,11 +1,15 @@ import os from pathlib import Path +import pytest + from config.config_manager import ( DEFAULT_EXCLUDED_DIRS, DEFAULT_EXCLUDED_EXTENSIONS, DEFAULT_EXCLUDED_FILES, ConfigManager, + parse_library_structure, + parse_platform_structures, ) @@ -79,6 +83,22 @@ def test_config_loader(): assert loader.config.SCAN_LANGUAGE_PRIORITY == ["jp", "es"] assert loader.config.GAMELIST_MEDIA_THUMBNAIL == "box3d" assert loader.config.GAMELIST_MEDIA_IMAGE == "title_screen" + assert loader.config.STRUCTURE_TEMPLATES == { + "psx": "{category}/{gameDir}", + "nes": ["{gameFile}", "{category}/{gameFile}"], + } + # The accessor parses templates on demand; unset platforms get None. + psx = loader.config.platform_structure("psx") + assert psx is not None and len(psx) == 1 + assert psx[0].each_file_is_game is False + assert len(psx[0].levels) == 1 and psx[0].levels[0].literal is None + # The list form yields one structure per template (union on discovery). + nes = loader.config.platform_structure("nes") + assert nes is not None and len(nes) == 2 + assert nes[0].each_file_is_game is True and nes[0].levels == () + assert nes[1].each_file_is_game is True + assert len(nes[1].levels) == 1 and nes[1].levels[0].literal is None + assert loader.config.platform_structure("snes") is None def test_empty_config_loader(): @@ -113,6 +133,59 @@ def test_empty_config_loader(): assert loader.config.EJS_CONTROLS == {} assert loader.config.GAMELIST_MEDIA_THUMBNAIL == "box2d" assert loader.config.GAMELIST_MEDIA_IMAGE == "screenshot" + assert loader.config.STRUCTURE_TEMPLATES == {} + + +@pytest.mark.parametrize( + ("template", "levels", "each_file_is_game"), + [ + ("{gameFile}", (), True), + ("{gameDir}", (), False), + ("{category}/{gameFile}", (None,), True), + ("Hacks/{gameFile}", ("Hacks",), True), + ("{region}/{system}/{gameDir}", (None, None), False), + ("roms/{region}/{gameFile}", ("roms", None), True), + ], +) +def test_parse_library_structure_valid(template, levels, each_file_is_game): + structure = parse_library_structure(template) + assert structure.each_file_is_game is each_file_is_game + assert tuple(level.literal for level in structure.levels) == levels + + +@pytest.mark.parametrize( + "template", + [ + "", + "justliteral", + "{category}", # no terminal + "{gameFile}/{gameDir}", # terminal not last + "{gameDir}/extra", # terminal not last + "{platform}/{gameFile}", # reserved macro RomM resolves itself + "{library}/{gameFile}", + "{}/{gameFile}", # empty macro + ], +) +def test_parse_library_structure_invalid(template): + with pytest.raises(ValueError): + parse_library_structure(template) + + +def test_parse_platform_structures_string_and_list(): + # A bare string yields a single structure. + single = parse_platform_structures("{gameFile}") + assert len(single) == 1 and single[0].each_file_is_game is True + + # A list yields one structure per template, preserving order. + multi = parse_platform_structures(["{gameFile}", "{category}/{gameDir}"]) + assert len(multi) == 2 + assert multi[0].levels == () and multi[0].each_file_is_game is True + assert len(multi[1].levels) == 1 and multi[1].each_file_is_game is False + + +def test_parse_platform_structures_propagates_invalid(): + with pytest.raises(ValueError): + parse_platform_structures(["{gameFile}", "nope"]) def test_missing_config_file_is_created(tmp_path): diff --git a/backend/tests/endpoints/sockets/test_scan.py b/backend/tests/endpoints/sockets/test_scan.py index abac05e2a9..9c1929993f 100644 --- a/backend/tests/endpoints/sockets/test_scan.py +++ b/backend/tests/endpoints/sockets/test_scan.py @@ -1,12 +1,21 @@ +import hashlib +from pathlib import Path from unittest.mock import Mock import pytest import socketio -from endpoints.sockets.scan import ScanStats, _should_scan_rom -from handler.filesystem.roms_handler import FSRomsHandler +from endpoints.sockets.scan import ( + ScanStats, + _reconcile_relocated_roms, + _should_scan_rom, +) +from handler.database import db_rom_handler +from handler.filesystem import fs_rom_handler +from handler.filesystem.roms_handler import FSRom, FSRomsHandler from handler.metadata.base_handler import UniversalPlatformSlug as UPS from handler.scan_handler import ScanType +from models.platform import Platform from models.rom import Rom @@ -326,3 +335,97 @@ def test_url_contains_fs_path_and_name(self, handler: FSRomsHandler): assert url is not None assert fs_path in url assert fs_name in url + + +def _fs_rom(fs_name: str, fs_path: str) -> FSRom: + return FSRom( + fs_name=fs_name, + fs_path=fs_path, + flat=True, + nested=False, + files=[], + crc_hash="", + md5_hash="", + sha1_hash="", + ra_hash="", + ) + + +class TestReconcileRelocatedRoms: + """A rom whose on-disk path changed (custom library structure) is relocated + in place by content hash instead of being re-imported as new, so its DB row + — and the saves/history/favorites/collections attached to it — survives.""" + + @pytest.mark.asyncio + async def test_moved_file_is_relocated_not_reimported( + self, platform: Platform, tmp_path: Path, monkeypatch + ): + content = b"relocate me please" + sha1 = hashlib.sha1(content, usedforsecurity=False).hexdigest() + base = f"{platform.fs_slug}/roms" + + # An existing rom recorded at the platform root, with its content hash. + rom = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="Mover", + slug="mover", + fs_name="Mover.bin", + fs_path=base, + sha1_hash=sha1, + fs_size_bytes=len(content), + ) + ) + + # On disk the file now lives inside a nested folder (same bytes). + sub = tmp_path / base / "Hacks" + sub.mkdir(parents=True) + (sub / "Mover.bin").write_bytes(content) + monkeypatch.setattr(fs_rom_handler, "base_path", tmp_path) + + fs_roms = [_fs_rom("Mover.bin", f"{base}/Hacks")] + handled = await _reconcile_relocated_roms(platform, fs_roms) + + # The moved file is reported handled, so the scan loop skips it. + assert handled == {f"{base}/Hacks/Mover.bin"} + + # Same row, new path, present again — not a fresh import. + all_roms = db_rom_handler.get_roms_for_relocation(platform.id) + assert len(all_roms) == 1 + updated = db_rom_handler.get_rom(rom.id) + assert updated is not None + assert updated.id == rom.id + assert updated.fs_path == f"{base}/Hacks" + assert updated.fs_name == "Mover.bin" + assert updated.missing_from_fs is False + + @pytest.mark.asyncio + async def test_no_hash_match_is_left_for_normal_import( + self, platform: Platform, tmp_path: Path, monkeypatch + ): + base = f"{platform.fs_slug}/roms" + # Existing rom whose stored hash won't match anything on disk. + db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="Other", + slug="other", + fs_name="Other.bin", + fs_path=base, + sha1_hash=hashlib.sha1(b"unrelated", usedforsecurity=False).hexdigest(), + fs_size_bytes=len(b"unrelated"), + ) + ) + + content = b"a brand new game entirely" + sub = tmp_path / base / "Hacks" + sub.mkdir(parents=True) + (sub / "New.bin").write_bytes(content) + monkeypatch.setattr(fs_rom_handler, "base_path", tmp_path) + + handled = await _reconcile_relocated_roms( + platform, [_fs_rom("New.bin", f"{base}/Hacks")] + ) + + # Nothing relocated: the new file falls through to a normal import. + assert handled == set() diff --git a/backend/tests/handler/filesystem/test_roms_handler.py b/backend/tests/handler/filesystem/test_roms_handler.py index 575f002f80..cc8b19d5ab 100644 --- a/backend/tests/handler/filesystem/test_roms_handler.py +++ b/backend/tests/handler/filesystem/test_roms_handler.py @@ -377,6 +377,223 @@ async def test_get_roms(self, handler: FSRomsHandler, platform, config): # Check excluded files are not present assert "excluded_test.tmp" not in rom_names + def _make_structure_config(self, templates: dict[str, str]) -> Config: + return Config( + EXCLUDED_PLATFORMS=[], + EXCLUDED_SINGLE_EXT=["tmp"], + EXCLUDED_SINGLE_FILES=[], + EXCLUDED_MULTI_FILES=["@eaDir"], + EXCLUDED_MULTI_PARTS_EXT=["tmp"], + EXCLUDED_MULTI_PARTS_FILES=[], + PLATFORMS_BINDING={}, + PLATFORMS_VERSIONS={}, + ROMS_FOLDER_NAME="roms", + FIRMWARE_FOLDER_NAME="bios", + STRUCTURE_TEMPLATES=templates, + ) + + def _build_structure_library(self, tmp_path: Path, base: str) -> None: + """Library exercising literal/wildcard levels, file-vs-folder terminals, + a name collision across folders, a hidden folder, and depth > 1.""" + roms = tmp_path / base + roms.mkdir(parents=True) + (roms / "Top.zip").write_text("t") + hacks = roms / "Hacks" + hacks.mkdir() + (hacks / "Shared.zip").write_text("a") + (hacks / "HackOnly.zip").write_text("b") + inner = hacks / "Inner" + inner.mkdir() + (inner / "x.bin").write_text("x") + trans = roms / "Translations" + trans.mkdir() + (trans / "Shared.zip").write_text("c") # collides with Hacks/Shared.zip + hidden = roms / ".hidden" + hidden.mkdir() + (hidden / "secret.zip").write_text("s") + region = roms / "Region" + region.mkdir() + usa = region / "USA" + usa.mkdir() + (usa / "usagame.zip").write_text("u") + + @pytest.mark.asyncio + async def test_get_roms_no_structure_is_default( + self, platform: Platform, tmp_path: Path + ): + """Without a template, only top-level files and folders are surfaced + (each file a flat rom, each folder a single multi-file rom).""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config({}), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (base, "Top.zip") in keys + assert (base, "Hacks") in keys + assert (base, "Translations") in keys + assert (base, "Region") in keys + assert (base, ".hidden") in keys + # Nothing nested is surfaced. + assert all(r["fs_path"] == base for r in roms) + assert len(roms) == 5 + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_wildcard_file_terminal( + self, platform: Platform, tmp_path: Path + ): + """`{category}/{gameFile}` descends one wildcard level and treats each + file as a rom; hidden folders are skipped and identically-named files in + different folders stay distinct via their full path.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config( + {platform.fs_slug: "{category}/{gameFile}"} + ), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + assert (f"{base}/Translations", "Shared.zip") in keys # collision kept + # Top-level file isn't surfaced (template requires one level down). + assert (base, "Top.zip") not in keys + # A folder at the terminal level isn't a {gameFile} rom. + assert (f"{base}/Hacks", "Inner") not in keys + # Hidden folder is never descended into. + assert not any(r["fs_path"].endswith("/.hidden") for r in roms) + full_paths = [f"{r['fs_path']}/{r['fs_name']}" for r in roms] + assert len(full_paths) == len(set(full_paths)) + assert sum(1 for r in roms if r["fs_name"] == "Shared.zip") == 2 + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_wildcard_dir_terminal( + self, platform: Platform, tmp_path: Path + ): + """`{category}/{gameDir}` treats each folder at the terminal level as a + single multi-file rom (kept whole — no disc-splitting guesswork).""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config( + {platform.fs_slug: "{category}/{gameDir}"} + ), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + # Folders at the terminal level become whole multi-file roms... + assert (f"{base}/Hacks", "Inner") in keys + assert (f"{base}/Region", "USA") in keys + # ...and are not split into their contents. + assert (f"{base}/Hacks/Inner", "x.bin") not in keys + # Files at the terminal level are not {gameDir} roms. + assert (f"{base}/Hacks", "Shared.zip") not in keys + assert all(r["nested"] for r in roms) + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_two_wildcard_levels( + self, platform: Platform, tmp_path: Path + ): + """Multiple wildcard levels descend depth-first to the terminal.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config( + {platform.fs_slug: "{region}/{system}/{gameFile}"} + ), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (f"{base}/Hacks/Inner", "x.bin") in keys + assert (f"{base}/Region/USA", "usagame.zip") in keys + # Depth-1 files are not surfaced at a depth-2 terminal. + assert (f"{base}/Hacks", "Shared.zip") not in keys + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_literal_level( + self, platform: Platform, tmp_path: Path + ): + """A literal section matches that exact folder only; sibling folders are + ignored.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config({platform.fs_slug: "Hacks/{gameFile}"}), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + # The literal only matched "Hacks"; "Translations" is ignored. + assert (f"{base}/Translations", "Shared.zip") not in keys + assert all(r["fs_path"] == f"{base}/Hacks" for r in roms) + assert count == len(roms) + + @pytest.mark.asyncio + async def test_get_roms_structure_list_unions_loose_and_grouped( + self, platform: Platform, tmp_path: Path + ): + """A list of templates unions their discovery: loose top-level games + (`{gameFile}`) plus games inside grouping subfolders + (`{category}/{gameFile}`) — the common mixed layout — without dropping + either, deduplicated by full path.""" + handler = FSRomsHandler() + handler.base_path = tmp_path + with patch( + "handler.filesystem.roms_handler.cm.get_config", + lambda: self._make_structure_config( + {platform.fs_slug: ["{gameFile}", "{category}/{gameFile}"]} + ), + ): + base = handler.get_roms_fs_structure(platform.fs_slug) + self._build_structure_library(tmp_path, base) + roms = await handler.get_roms(platform) + count = await handler.count_roms(platform) + + keys = {(r["fs_path"], r["fs_name"]) for r in roms} + # Loose top-level game ({gameFile}). + assert (base, "Top.zip") in keys + # Grouped games one level down ({category}/{gameFile}). + assert (f"{base}/Hacks", "Shared.zip") in keys + assert (f"{base}/Hacks", "HackOnly.zip") in keys + assert (f"{base}/Translations", "Shared.zip") in keys + # Hidden folder still skipped; no duplicate full paths. + assert not any(r["fs_path"].endswith("/.hidden") for r in roms) + full_paths = [f"{r['fs_path']}/{r['fs_name']}" for r in roms] + assert len(full_paths) == len(set(full_paths)) + assert count == len(roms) + @pytest.mark.asyncio async def test_get_rom_files_single_rom( self, handler: FSRomsHandler, rom_single, config diff --git a/backend/tests/handler/test_db_handler.py b/backend/tests/handler/test_db_handler.py index 35e89d7a67..1561c8eebf 100644 --- a/backend/tests/handler/test_db_handler.py +++ b/backend/tests/handler/test_db_handler.py @@ -1,6 +1,7 @@ from datetime import datetime, timezone import pytest +from sqlalchemy import inspect as sa_inspect from sqlalchemy.exc import IntegrityError from config import ROMM_DB_DRIVER @@ -544,6 +545,38 @@ def test_bulk_mark_present_chunking(platform: Platform): assert updated.missing_from_fs is False +def test_get_roms_by_fs_name_keys_on_full_path(platform: Platform): + """Identically-named files in different folders must stay distinct: the + result is keyed on full path (fs_path/fs_name), not just the file name.""" + root = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="dup_root", + slug="dup-root", + fs_name="Game.zip", + fs_path=f"{platform.slug}/roms", + ) + ) + nested = db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name="dup_nested", + slug="dup-nested", + fs_name="Game.zip", + fs_path=f"{platform.slug}/roms/Hacks", + ) + ) + + result = db_rom_handler.get_roms_by_fs_name(platform.id, ["Game.zip"]) + + assert set(result) == { + f"{platform.slug}/roms/Game.zip", + f"{platform.slug}/roms/Hacks/Game.zip", + } + assert result[f"{platform.slug}/roms/Game.zip"].id == root.id + assert result[f"{platform.slug}/roms/Hacks/Game.zip"].id == nested.id + + def test_mark_missing_roms_small_platform(platform: Platform): """mark_missing_roms correctly identifies missing ROMs with a small keep list.""" rom_a = db_rom_handler.add_rom( @@ -583,12 +616,22 @@ def test_mark_missing_roms_small_platform(platform: Platform): ) ) - # Keep only rom_a and rom_c - missing = db_rom_handler.mark_missing_roms(platform.id, ["rom_a.zip", "rom_c.zip"]) + # Keep only rom_a and rom_c (keep-list holds full paths) + missing = db_rom_handler.mark_missing_roms( + platform.id, + [f"{platform.slug}/roms/rom_a.zip", f"{platform.slug}/roms/rom_c.zip"], + ) assert len(missing) == 1 assert missing[0].fs_name == "rom_b.zip" + # Regression: returned (detached) instances must carry `fs_path` so callers + # can read `rom.full_path` after the session closes without a + # DetachedInstanceError (the missing-rom warning in scan.py does exactly + # that). + assert "fs_path" not in sa_inspect(missing[0]).unloaded + assert missing[0].full_path == f"{platform.slug}/roms/rom_b.zip" + updated_b = db_rom_handler.get_rom(rom_b.id) assert updated_b is not None assert updated_b.missing_from_fs is True @@ -627,7 +670,9 @@ def test_mark_missing_roms_large_platform(platform: Platform): # Build a large keep list to verify mark_missing_roms() handles many entries. # Only rom_present.zip actually exists in DB; the rest are just filler. - fs_roms_to_keep = ["rom_present.zip"] + [f"filler_{i}.zip" for i in range(501)] + fs_roms_to_keep = [f"{platform.slug}/roms/rom_present.zip"] + [ + f"filler_{i}.zip" for i in range(501) + ] missing = db_rom_handler.mark_missing_roms(platform.id, fs_roms_to_keep) @@ -662,7 +707,7 @@ def test_mark_missing_roms_large_platform_all_present(platform: Platform): roms.append(rom) # Keep list has all real ROMs plus filler to exceed 500 - fs_roms_to_keep = [f"rom_{i}.zip" for i in range(3)] + [ + fs_roms_to_keep = [f"{platform.slug}/roms/rom_{i}.zip" for i in range(3)] + [ f"filler_{i}.zip" for i in range(500) ] diff --git a/backend/tests/handler/test_fastapi.py b/backend/tests/handler/test_fastapi.py index 1e240e542f..d9d6eceebf 100644 --- a/backend/tests/handler/test_fastapi.py +++ b/backend/tests/handler/test_fastapi.py @@ -70,6 +70,7 @@ async def test_scan_rom(): rom=rom, fs_rom={ "fs_name": "Paper Mario (USA).z64", + "fs_path": "n64/Paper Mario (USA)", "flat": True, "nested": False, "files": [ @@ -160,6 +161,7 @@ async def test_scan_rom_complete_clears_unselected_metadata( rom=rom, fs_rom={ "fs_name": "Paper Mario (USA).z64", + "fs_path": "n64/Paper Mario (USA)", "flat": True, "nested": False, "files": [], @@ -230,6 +232,7 @@ async def test_scan_rom_unmatched_fetches_ra_when_id_set_but_no_metadata( rom=rom, fs_rom={ "fs_name": "Jak and Daxter.chd", + "fs_path": "ps2", "flat": True, "nested": False, "files": [], @@ -292,6 +295,7 @@ async def test_scan_rom_unmatched_skips_ra_when_id_and_metadata_exist( rom=rom, fs_rom={ "fs_name": "Jak and Daxter.chd", + "fs_path": "ps2", "flat": True, "nested": False, "files": [], diff --git a/examples/config.example.yml b/examples/config.example.yml index 8d539c0f09..a22c78fd19 100644 --- a/examples/config.example.yml +++ b/examples/config.example.yml @@ -76,6 +76,53 @@ # roms_folder: 'roms' # # Skip file hash calculations on low power devices (eg. Raspberry PI) # skip_hash_calculation: false +# # Custom library structure, per platform. By default a platform's ROM folder +# # is scanned one level deep: each top-level file is a game and each top-level +# # folder is a single multi-file game. A template lets you describe a deeper / +# # organized layout instead. It is opt-in per platform (by fs_slug); platforms +# # left out keep the default behavior. +# # +# # Syntax (Retrom-style): `/`-separated path sections, relative to the +# # platform's ROM folder (RomM already resolves the library root, the +# # roms_folder above, and the platform directory — so {library}/{platform} +# # are not used here). A section wrapped in braces is a macro; a bare section +# # is a literal folder name matched exactly. The LAST section must be a +# # terminal macro: +# # - {gameFile} -> each file at that level is its own game; +# # - {gameDir} -> each folder at that level is a single multi-file game. +# # Any other braced section (e.g. {region}, {category}) is a wildcard +# # directory level: it matches any folder and is purely organizational. +# # +# # A platform may also declare a LIST of templates instead of one; discovery +# # is their union. This covers the common mixed layout — loose games at the +# # platform root AND games inside grouping subfolders below it — which a +# # single fixed-depth template can't express on its own. +# # +# # Notes: +# # - Hidden (dot-prefixed) folders are never descended into or surfaced. +# # - {gameDir} keeps a folder whole (multi-disc / cue+bin games), so there +# # is no guessing — you declare file-vs-folder explicitly. +# # - Identity is content-based on scan: moving or renaming a game within the +# # structure is detected by its hash and relocated in place, so its saves, +# # play history, favorites and collection membership follow it (as long as +# # hashing is enabled and the platform is hashable). When a folder that was +# # previously scanned as one multi-file game is now descended into, the old +# # entry is marked "missing from filesystem" (the scan log flags it); +# # delete the stale entry to clean up. +# # - Two files with the same name in different folders become distinct games; +# # gamelist.xml metadata matching is by filename, so both may match the +# # same gamelist entry. +# structure: +# # Mixed: loose top-level games AND grouping subfolders below them, e.g. +# # roms/nes/game01.nes (matched by {gameFile}) +# # roms/nes/Hacks/game03.nes (matched by {category}/{gameFile}) +# nes: +# - '{gameFile}' +# - '{category}/{gameFile}' +# # roms/ps3/Disc/Game/, roms/ps3/PSN/Game/ -> each folder a multi-file game +# ps3: '{category}/{gameDir}' +# # roms/snes/USA/foo.sfc, roms/snes/Japan/bar.sfc +# snes: '{region}/{gameFile}' # scan: # # Metadata priority during scans diff --git a/frontend/src/locales/bg_BG/rom.json b/frontend/src/locales/bg_BG/rom.json index 1f4a6bfc59..250cf0bc19 100644 --- a/frontend/src/locales/bg_BG/rom.json +++ b/frontend/src/locales/bg_BG/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Местоположение", + "location-copied": "Местоположението е копирано в клипборда." } diff --git a/frontend/src/locales/cs_CZ/rom.json b/frontend/src/locales/cs_CZ/rom.json index 03b8263dc8..bf1a465a2d 100644 --- a/frontend/src/locales/cs_CZ/rom.json +++ b/frontend/src/locales/cs_CZ/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Umístění", + "location-copied": "Umístění zkopírováno do schránky." } diff --git a/frontend/src/locales/de_DE/rom.json b/frontend/src/locales/de_DE/rom.json index 32cb1c6301..88b8006eb8 100644 --- a/frontend/src/locales/de_DE/rom.json +++ b/frontend/src/locales/de_DE/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Speicherort", + "location-copied": "Speicherort in die Zwischenablage kopiert." } diff --git a/frontend/src/locales/en_GB/rom.json b/frontend/src/locales/en_GB/rom.json index b8f4312209..6b9ce4aee1 100644 --- a/frontend/src/locales/en_GB/rom.json +++ b/frontend/src/locales/en_GB/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Location", + "location-copied": "Location copied to clipboard." } diff --git a/frontend/src/locales/en_US/rom.json b/frontend/src/locales/en_US/rom.json index f2532066c3..f20aa6e59b 100644 --- a/frontend/src/locales/en_US/rom.json +++ b/frontend/src/locales/en_US/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Location", + "location-copied": "Location copied to clipboard." } diff --git a/frontend/src/locales/es_ES/rom.json b/frontend/src/locales/es_ES/rom.json index cd5c798546..73d0c815a1 100644 --- a/frontend/src/locales/es_ES/rom.json +++ b/frontend/src/locales/es_ES/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Comunidad", "states-section-mine": "Mis estados", "states-section-community": "Comunidad", - "cant-toggle-visibility": "No se pudo cambiar la visibilidad: {error}" + "cant-toggle-visibility": "No se pudo cambiar la visibilidad: {error}", + "location": "Ubicación", + "location-copied": "Ubicación copiada al portapapeles." } diff --git a/frontend/src/locales/fr_FR/rom.json b/frontend/src/locales/fr_FR/rom.json index 795c525d6f..a62b5b7521 100644 --- a/frontend/src/locales/fr_FR/rom.json +++ b/frontend/src/locales/fr_FR/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Emplacement", + "location-copied": "Emplacement copié dans le presse-papiers." } diff --git a/frontend/src/locales/hu_HU/rom.json b/frontend/src/locales/hu_HU/rom.json index d723399159..e54a11fcb9 100644 --- a/frontend/src/locales/hu_HU/rom.json +++ b/frontend/src/locales/hu_HU/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Hely", + "location-copied": "A hely a vágólapra másolva." } diff --git a/frontend/src/locales/it_IT/rom.json b/frontend/src/locales/it_IT/rom.json index 39e2dfc172..bddcaf1127 100644 --- a/frontend/src/locales/it_IT/rom.json +++ b/frontend/src/locales/it_IT/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Posizione", + "location-copied": "Posizione copiata negli appunti." } diff --git a/frontend/src/locales/ja_JP/rom.json b/frontend/src/locales/ja_JP/rom.json index 3af098cc99..80c1ba91b1 100644 --- a/frontend/src/locales/ja_JP/rom.json +++ b/frontend/src/locales/ja_JP/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "場所", + "location-copied": "場所をクリップボードにコピーしました。" } diff --git a/frontend/src/locales/ko_KR/rom.json b/frontend/src/locales/ko_KR/rom.json index f69330a07f..5e39f4f447 100644 --- a/frontend/src/locales/ko_KR/rom.json +++ b/frontend/src/locales/ko_KR/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "위치", + "location-copied": "위치를 클립보드에 복사했습니다." } diff --git a/frontend/src/locales/pl_PL/rom.json b/frontend/src/locales/pl_PL/rom.json index 35acce1ad3..3dc1405a97 100644 --- a/frontend/src/locales/pl_PL/rom.json +++ b/frontend/src/locales/pl_PL/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Lokalizacja", + "location-copied": "Skopiowano lokalizację do schowka." } diff --git a/frontend/src/locales/pt_BR/rom.json b/frontend/src/locales/pt_BR/rom.json index 7277b32792..fbd13e58f2 100644 --- a/frontend/src/locales/pt_BR/rom.json +++ b/frontend/src/locales/pt_BR/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Localização", + "location-copied": "Localização copiada para a área de transferência." } diff --git a/frontend/src/locales/ro_RO/rom.json b/frontend/src/locales/ro_RO/rom.json index b330533ae9..5586553ae4 100644 --- a/frontend/src/locales/ro_RO/rom.json +++ b/frontend/src/locales/ro_RO/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Locație", + "location-copied": "Locație copiată în clipboard." } diff --git a/frontend/src/locales/ru_RU/rom.json b/frontend/src/locales/ru_RU/rom.json index 87f10c6970..ff960b6cd8 100644 --- a/frontend/src/locales/ru_RU/rom.json +++ b/frontend/src/locales/ru_RU/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "Расположение", + "location-copied": "Расположение скопировано в буфер обмена." } diff --git a/frontend/src/locales/zh_CN/rom.json b/frontend/src/locales/zh_CN/rom.json index e312b7628f..4c43735da4 100644 --- a/frontend/src/locales/zh_CN/rom.json +++ b/frontend/src/locales/zh_CN/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "位置", + "location-copied": "位置已复制到剪贴板。" } diff --git a/frontend/src/locales/zh_TW/rom.json b/frontend/src/locales/zh_TW/rom.json index a39faedf53..66b6be4795 100644 --- a/frontend/src/locales/zh_TW/rom.json +++ b/frontend/src/locales/zh_TW/rom.json @@ -412,5 +412,7 @@ "saves-section-community": "Community", "states-section-mine": "My states", "states-section-community": "Community", - "cant-toggle-visibility": "Could not change visibility: {error}" + "cant-toggle-visibility": "Could not change visibility: {error}", + "location": "位置", + "location-copied": "位置已複製到剪貼簿。" } diff --git a/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue b/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue index 71beee439d..5518a518b4 100644 --- a/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue +++ b/frontend/src/v2/components/GameDetails/FilesTab/FilesSummary.vue @@ -7,6 +7,7 @@ import { computed } from "vue"; import type { DetailedRomSchema } from "@/__generated__"; import { formatBytes } from "@/utils"; import HashChip from "@/v2/components/shared/HashChip.vue"; +import LocationChip from "@/v2/components/shared/LocationChip.vue"; import MissingFSBadge from "@/v2/components/shared/MissingFSBadge.vue"; defineOptions({ inheritAttrs: false }); @@ -15,6 +16,12 @@ const props = defineProps<{ rom: DetailedRomSchema }>(); const fileCount = computed(() => props.rom.files?.length ?? 0); +// Full on-disk path of the ROM — its directory plus its name. Surfaces +// *where* a ROM sits in the library (e.g. a platform subfolder like +// `roms/nes/Hacks/…`), which is otherwise only implicit. Distinct from +// the per-file relative paths shown in the file list below. +const fullPath = computed(() => `${props.rom.fs_path}/${props.rom.fs_name}`); + interface RomHash { label: string; value: string | null; @@ -57,6 +64,10 @@ const hashes = computed(() => { +
+ +
+
(() => { .r-v2-files-summary__sep { opacity: 0.5; } +.r-v2-files-summary__location { + display: flex; + min-width: 0; +} .r-v2-files-summary__hashes { display: flex; align-items: center; diff --git a/frontend/src/v2/components/shared/LocationChip.vue b/frontend/src/v2/components/shared/LocationChip.vue new file mode 100644 index 0000000000..5f1d5414c7 --- /dev/null +++ b/frontend/src/v2/components/shared/LocationChip.vue @@ -0,0 +1,82 @@ + + + + +