feat(cloud-sync): support config/thumbnails/system as opaque per-user blobs

backend/config/__init__.py

@@ -36,6 +36,10 @@ def _get_env(var: str, fallback: str | None = None) -> str | None:
 LIBRARY_BASE_PATH: Final[str] = f"{ROMM_BASE_PATH}/library"
 RESOURCES_BASE_PATH: Final[str] = f"{ROMM_BASE_PATH}/resources"
 ASSETS_BASE_PATH: Final[str] = f"{ROMM_BASE_PATH}/assets"
+# Opaque storage for RetroArch Cloud Sync categories RomM has no concept of
+# (config/, thumbnails/, system/) — unrelated to any ROM, so it lives outside
+# the asset tree but still under the same persistent volume.
+CLOUD_SYNC_BLOB_BASE_PATH: Final[str] = f"{ROMM_BASE_PATH}/cloud_sync_blobs"
 ZIP_CACHE_PATH: Final[str] = f"{ROMM_BASE_PATH}/cache/zips"
 FRONTEND_RESOURCES_PATH: Final[str] = "/assets/romm/resources"
 

backend/endpoints/cloud_sync.py

@@ -10,6 +10,8 @@
 heap.
 """
 
+import os
+
 from fastapi import APIRouter, Request, Response, status
 from fastapi.responses import JSONResponse
 
@@ -18,7 +20,7 @@
 from handler.auth.dependencies import get_permissions
 from handler.cloud_sync_handler import MANIFEST_FILE_NAME, AssetKind, CloudSyncPath
 from handler.database import db_save_handler, db_state_handler
-from handler.filesystem import fs_asset_handler
+from handler.filesystem import fs_asset_handler, fs_cloud_sync_blob_handler
 from handler.filesystem.assets_handler import build_asset_file_response
 from handler.scan_handler import scan_save, scan_state
 from logger.formatter import BLUE
@@ -121,6 +123,22 @@ async def cloud_sync_get(request: Request, file_path: str) -> Response:
         )
         return JSONResponse(content=manifest)
 
+    blob_path = cloud_sync_handler.parse_cloud_sync_blob_path(file_path)
+    if blob_path:
+        try:
+            resolved_path = fs_cloud_sync_blob_handler.validate_path(
+                cloud_sync_handler.user_blob_path(request.user, blob_path)
+            )
+        except ValueError:
+            return _empty(status.HTTP_404_NOT_FOUND)
+
+        if not resolved_path.is_file():
+            return _empty(status.HTTP_404_NOT_FOUND)
+
+        return build_asset_file_response(
+            resolved_path, filename=os.path.basename(blob_path)
+        )
+
     parsed = cloud_sync_handler.parse_cloud_sync_path(file_path)
     if not parsed:
         return _empty(status.HTTP_404_NOT_FOUND)
@@ -157,9 +175,23 @@ async def cloud_sync_put(request: Request, file_path: str) -> Response:
         return _empty(status.HTTP_204_NO_CONTENT)
 
     # RetroArch also offers config/, thumbnails/ and system/ when those settings
-    # are on. RomM has nowhere to put opaque blobs, and answering OK would put
-    # them in the client's local manifest, so the next sync would read their
-    # absence from ours as a remote delete and wipe the local copies.
+    # are on. None of these belong to a ROM, so they're stored as opaque
+    # per-user blobs instead of going through the asset/ROM matching below.
+    blob_path = cloud_sync_handler.parse_cloud_sync_blob_path(file_path)
+    if blob_path:
+        disk_path = cloud_sync_handler.user_blob_path(request.user, blob_path)
+        existed = await fs_cloud_sync_blob_handler.file_exists(disk_path)
+
+        await fs_cloud_sync_blob_handler.write_file(
+            file=await request.body(),
+            path=os.path.dirname(disk_path),
+            filename=os.path.basename(disk_path),
+        )
+
+        return _empty(
+            status.HTTP_204_NO_CONTENT if existed else status.HTTP_201_CREATED
+        )
+
     parsed = cloud_sync_handler.parse_cloud_sync_path(file_path)
     if not parsed:
         return _empty(status.HTTP_409_CONFLICT)
@@ -248,6 +280,17 @@ async def cloud_sync_delete(request: Request, file_path: str) -> Response:
     if denied:
         return denied
 
+    blob_path = cloud_sync_handler.parse_cloud_sync_blob_path(file_path)
+    if blob_path:
+        try:
+            await fs_cloud_sync_blob_handler.remove_file(
+                file_path=cloud_sync_handler.user_blob_path(request.user, blob_path)
+            )
+        except FileNotFoundError:
+            return _empty(status.HTTP_404_NOT_FOUND)
+
+        return _empty(status.HTTP_204_NO_CONTENT)
+
     parsed = cloud_sync_handler.parse_cloud_sync_path(file_path)
     if not parsed:
         return _empty(status.HTTP_404_NOT_FOUND)

backend/handler/cloud_sync_emulator_names.py

@@ -0,0 +1,98 @@
+"""Translates between RomM's `emulator` field convention (lowercase libretro
+core identifier, e.g. "snes9x") and RetroArch's actual local save/state
+directory name (its display name, e.g. "Snes9x"). These are not the same
+string, and naively round-tripping one as the other has two failure modes:
+
+- Writing RetroArch's raw folder name straight into `emulator` stores a save
+  RomM's own web player can never select again -- `EmulatorJS.vue` filters
+  saves by an exact match against the lowercase libretro core id, and
+  `_EJS_CORES_MAP` in the frontend confirms that convention is RomM-wide, not
+  cloud-sync-specific.
+- Handing that raw value back out unchanged in the manifest can point
+  RetroArch at a folder its own local install never uses (it's case- and
+  spacing-sensitive), so the file silently never resolves as "already
+  synced" and keeps re-appearing as a diff.
+
+The table mirrors the community romm-retroarch-sync project
+(github.com/Covin90/romm-retroarch-sync), which had already solved this
+exact problem for the cores below. Anything outside the table round-trips
+unchanged on the way back out to RetroArch rather than guessing at a casing
+or spacing that hasn't been verified against a real install.
+"""
+
+RETROARCH_DIR_BY_ROMM_EMULATOR: dict[str, str] = {
+    # SNES
+    "snes9x": "Snes9x",
+    "bsnes": "bsnes",
+    "mesen-s": "Mesen-S",
+    # NES
+    "nestopia": "Nestopia",
+    "fceumm": "FCEUmm",
+    "mesen": "Mesen",
+    # PlayStation
+    "beetle_psx": "Beetle PSX",
+    "beetle_psx_hw": "Beetle PSX HW",
+    "pcsx_rearmed": "PCSX-ReARMed",
+    "swanstation": "SwanStation",
+    "mednafen_psx": "Beetle PSX",
+    "mednafen_psx_hw": "Beetle PSX HW",
+    # Game Boy
+    "gambatte": "Gambatte",
+    "sameboy": "SameBoy",
+    "tgbdual": "TGB Dual",
+    "mgba": "mGBA",
+    "vba_next": "VBA Next",
+    "vbam": "VBA-M",
+    # Genesis / Mega Drive
+    "genesis_plus_gx": "Genesis Plus GX",
+    "blastem": "BlastEm",
+    "picodrive": "PicoDrive",
+    # Nintendo 64
+    "mupen64plus_next": "Mupen64Plus-Next",
+    "parallel_n64": "ParaLLEl N64",
+    # Saturn
+    "beetle_saturn": "Beetle Saturn",
+    "kronos": "Kronos",
+    "mednafen_saturn": "Beetle Saturn",
+    # Arcade / Neo Geo
+    "mame": "MAME",
+    "fbneo": "FBNeo",
+    "fbalpha": "FB Alpha",
+    # PlayStation 2 / GameCube
+    "pcsx2": "PCSX2",
+    "play": "Play!",
+    "dolphin": "Dolphin",
+    # Dreamcast
+    "flycast": "Flycast",
+    "redream": "Redream",
+    # Atari
+    "stella": "Stella",
+    # PC Engine
+    "beetle_pce": "Beetle PCE",
+    "beetle_pce_fast": "Beetle PCE Fast",
+    "mednafen_pce": "Beetle PCE",
+    "mednafen_pce_fast": "Beetle PCE Fast",
+    # Other common cores
+    "dosbox_pure": "DOSBox-Pure",
+    "scummvm": "ScummVM",
+    "ppsspp": "PPSSPP",
+    "desmume": "DeSmuME",
+    "melonds": "melonDS",
+    "citra": "Citra",
+}
+
+
+def to_romm_emulator(retroarch_dir_name: str) -> str:
+    """RetroArch's local directory name (e.g. "Snes9x") -> RomM's `emulator`
+    convention (e.g. "snes9x"). A plain, universally-safe normalization --
+    RomM's own convention is always lowercase with underscores, so this
+    never needs a lookup table."""
+    return retroarch_dir_name.lower().replace(" ", "_").replace("-", "_")
+
+
+def to_retroarch_dir_name(romm_emulator: str) -> str:
+    """RomM's `emulator` value -> RetroArch's local directory name. Cores
+    outside the table round-trip unchanged: guessing at a casing or spacing
+    that hasn't been verified against a real RetroArch install risks
+    inventing a folder that's just as wrong as the untranslated one."""
+    return RETROARCH_DIR_BY_ROMM_EMULATOR.get(romm_emulator.lower(), romm_emulator)

backend/handler/cloud_sync_handler.py

@@ -17,8 +17,9 @@
 from dataclasses import dataclass
 from typing import Literal
 
+from handler.cloud_sync_emulator_names import to_retroarch_dir_name, to_romm_emulator
 from handler.database import db_rom_handler, db_save_handler, db_state_handler
-from handler.filesystem import fs_asset_handler
+from handler.filesystem import fs_asset_handler, fs_cloud_sync_blob_handler
 from handler.redis_handler import async_cache
 from models.assets import Save, State
 from models.rom import Rom
@@ -30,6 +31,12 @@
 
 ASSET_ROOTS: dict[str, AssetKind] = {"saves": "saves", "states": "states"}
 
+# RetroArch's other three Cloud Sync categories (Settings -> Saving -> Cloud
+# Sync -> Sync Configuration/Thumbnails/System Files). Unlike saves/states,
+# none of these belong to a ROM, so they're kept as opaque per-user blobs
+# instead of going through the asset/ROM matching machinery below.
+BLOB_CATEGORIES = ("config", "thumbnails", "system")
+
 # `<game>.state`, `<game>.state3`, `<game>.state.auto` — the auto suffix makes
 # this a two-segment extension, which splitext alone gets wrong.
 STATE_SUFFIX_PATTERN = re.compile(r"\.state\d*(?:\.auto)?$", re.IGNORECASE)
@@ -53,7 +60,11 @@ def parse_cloud_sync_path(path: str) -> CloudSyncPath | None:
     """Parse a client path, or None when it is not a supported asset path.
 
     Accepts ``<root>/<file>`` and ``<root>/<core>/<file>``; RetroArch produces
-    the latter when "sort saves into folders by core name" is on.
+    the latter when "sort saves into folders by core name" is on. The core
+    segment is RetroArch's own directory casing (e.g. "Snes9x"), normalized
+    here to RomM's `emulator` convention (e.g. "snes9x") -- storing it
+    unnormalized would make the save invisible to RomM's own web player,
+    which matches saves against the lowercase libretro core id.
     """
     segments = [segment for segment in path.strip("/").split("/") if segment]
     if not 2 <= len(segments) <= 3:
@@ -68,7 +79,7 @@ def parse_cloud_sync_path(path: str) -> CloudSyncPath | None:
 
     return CloudSyncPath(
         kind=kind,
-        emulator=segments[1] if len(segments) == 3 else None,
+        emulator=to_romm_emulator(segments[1]) if len(segments) == 3 else None,
         file_name=segments[-1],
     )
 
@@ -85,7 +96,7 @@ def game_name_from_file_name(kind: AssetKind, file_name: str) -> str:
 
 def build_cloud_sync_path(kind: AssetKind, emulator: str | None, file_name: str) -> str:
     if emulator:
-        return f"{kind}/{emulator}/{file_name}"
+        return f"{kind}/{to_retroarch_dir_name(emulator)}/{file_name}"
     return f"{kind}/{file_name}"
 
 
@@ -108,6 +119,74 @@ def build_asset_file_path(
     )
 
 
+def parse_cloud_sync_blob_path(path: str) -> str | None:
+    """A client path under one of the opaque blob categories, normalized to
+    a plain ``category/...`` posix string, or None if it isn't one.
+
+    Unlike asset paths these keep arbitrary nesting: RetroArch mirrors its
+    own on-device directory tree here (e.g. thumbnail packs are organized as
+    ``thumbnails/<system>/Named_Boxarts/<game>.png``), so there's no fixed
+    segment count to enforce.
+    """
+    segments = [segment for segment in path.strip("/").split("/") if segment]
+    if len(segments) < 2:
+        return None
+
+    if any(segment in (os.curdir, os.pardir) for segment in segments):
+        return None
+
+    if segments[0] not in BLOB_CATEGORIES:
+        return None
+
+    return "/".join(segments)
+
+
+def user_blob_path(user: User, blob_path: str) -> str:
+    """Where a parsed blob path lives on disk, namespaced by user so two
+    RetroArch installs syncing to the same RomM instance under different
+    accounts never see each other's config/thumbnails/system files."""
+    return f"{fs_asset_handler.user_folder_path(user)}/{blob_path}"
+
+
+async def blob_md5(user: User, blob_path: str) -> str | None:
+    try:
+        resolved = fs_cloud_sync_blob_handler.validate_path(
+            user_blob_path(user, blob_path)
+        )
+        stat = resolved.stat()
+    except (ValueError, OSError):
+        return None
+
+    cache_key = (
+        f"romm:cloud_sync:blob_md5:{user.id}:{blob_path}:{stat.st_size}:{stat.st_mtime}"
+    )
+    cached = await async_cache.get(cache_key)
+    if cached:
+        return cached.decode() if isinstance(cached, bytes) else str(cached)
+
+    digest = await fs_cloud_sync_blob_handler.compute_file_md5(
+        user_blob_path(user, blob_path)
+    )
+    if digest:
+        await async_cache.set(cache_key, digest, ex=_HASH_CACHE_TTL_SECONDS)
+
+    return digest
+
+
+async def build_blob_manifest_entries(user: User) -> list[dict[str, str]]:
+    entries: list[dict[str, str]] = []
+    for category in BLOB_CATEGORIES:
+        prefix = f"{fs_asset_handler.user_folder_path(user)}/{category}"
+        for relative in await fs_cloud_sync_blob_handler.list_blob_paths(prefix):
+            blob_path = f"{category}/{relative}"
+            digest = await blob_md5(user, blob_path)
+            if not digest:
+                continue
+            entries.append({"path": blob_path, "hash": digest})
+
+    return entries
+
+
 def resolve_rom(game_name: str, can_see: Callable[[Rom], bool]) -> Rom | None:
     """The ROM a cloud-sync file belongs to, matched on file name alone.
 
@@ -172,5 +251,7 @@ async def build_manifest(
             }
         )
 
+    entries += await build_blob_manifest_entries(user)
+
     entries.sort(key=lambda entry: entry["path"])
     return entries

backend/handler/filesystem/__init__.py

@@ -1,4 +1,5 @@
 from .assets_handler import FSAssetsHandler
+from .cloud_sync_blob_handler import FSCloudSyncBlobHandler
 from .firmware_handler import FSFirmwareHandler
 from .launchbox_handler import FSLaunchboxHandler, get_fs_launchbox_handler
 from .platforms_handler import FSPlatformsHandler
@@ -7,20 +8,23 @@
 from .sync_handler import FSSyncHandler, get_fs_sync_handler
 
 fs_asset_handler = FSAssetsHandler()
+fs_cloud_sync_blob_handler = FSCloudSyncBlobHandler()
 fs_firmware_handler = FSFirmwareHandler()
 fs_platform_handler = FSPlatformsHandler()
 fs_rom_handler = FSRomsHandler()
 fs_resource_handler = FSResourcesHandler()
 
 __all__ = [
     "FSAssetsHandler",
+    "FSCloudSyncBlobHandler",
     "FSFirmwareHandler",
     "FSLaunchboxHandler",
     "FSPlatformsHandler",
     "FSResourcesHandler",
     "FSRomsHandler",
     "FSSyncHandler",
     "fs_asset_handler",
+    "fs_cloud_sync_blob_handler",
     "fs_firmware_handler",
     "fs_platform_handler",
     "fs_resource_handler",