diff --git a/lib/import/import_container.dart b/lib/import/import_container.dart index 78fc62d5..2f12c948 100644 --- a/lib/import/import_container.dart +++ b/lib/import/import_container.dart @@ -529,10 +529,24 @@ Future resolveNoopDatabase(String path) async { 'of space. Free some up and try again.', ); } - if (db.size > 0 && written > db.size) { + // CRC is checked whenever the archive gives us one — not only on an + // over-run. A REAL NOOP export (10.5.0, measured 2026-08) has been seen + // shaped exactly like the over-run case below: the zip's own + // uncompressed-size field for `noop-backup.sqlite` undercounts the true + // content by a few thousand pages, while the archive's CRC-32 — + // computed over the FULL decompressed stream, not the declared length — + // matches what actually came out. That is a bug in whatever wrote the + // zip (its size field went stale before the deflate stream did), not a + // truncated or tampered file, and `unzip -t` reports the very same file + // as OK because it validates by CRC. Trust the CRC the same way — but + // an exact-size extraction can ALSO be silently wrong (same length, + // different bytes), which checking only `written > db.size` would miss + // entirely, so this runs for `written >= db.size`, not just the overrun. + final crc = db.crc32; + if (db.size > 0 && crc != null && await _fileCrc32(destPath) != crc) { throw ImportFormatException( '“${p.basename(path)}” does not hold what it says it does — its ' - 'database unpacked to more than the archive declared. It is likely ' + 'database does not match the archive checksum. It is likely ' 'damaged; export it again.', ); } @@ -553,6 +567,17 @@ Future resolveNoopDatabase(String path) async { } } +/// CRC-32 of a file's whole content, read in chunks — this runs on the +/// unpacked database (hundreds of MB to a couple of GB), and loading it whole +/// just to checksum it would double the memory an import already costs. +Future _fileCrc32(String path) async { + var crc = 0; + await for (final chunk in File(path).openRead()) { + crc = getCrc32(chunk, crc); + } + return crc; +} + /// Resolve the picked paths into CSV files on disk, unwrapping ZIP archives. /// /// [flavor] names the importer in error messages ('NOOP', 'WHOOP'). Extracted diff --git a/lib/import/noop_backup_import.dart b/lib/import/noop_backup_import.dart index 608ca9ce..1a161843 100644 --- a/lib/import/noop_backup_import.dart +++ b/lib/import/noop_backup_import.dart @@ -35,7 +35,9 @@ import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; +import 'package:path/path.dart' as p; import 'package:sqflite/sqflite.dart'; import '../compute/derivation_engine.dart'; @@ -44,6 +46,13 @@ import 'import_container.dart'; import 'noop_import.dart'; import 'noop_ingest.dart'; +/// "SQLite format 3\0" — the fixed 16-byte magic every SQLite file starts +/// with (https://www.sqlite.org/fileformat.html#the_database_header). +const List _kSqliteMagic = [ + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, // + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, +]; + /// Rows per platform-channel round trip. Big enough that a 1 Hz day is a handful /// of queries, small enough that no single response is a memory event. Not /// const: the page-boundary contract below is only testable at a small size. @@ -85,19 +94,113 @@ class NoopBackupImporter { if (!await File(path).exists()) { throw const ImportFormatException('That backup could not be read.'); } - final Database src; + Directory? scratch; try { - src = await openDatabase(path, readOnly: true); - } catch (e) { - throw ImportFormatException( - 'That backup\'s database could not be opened ($e). If it came off ' - 'another phone, try exporting it again.', - ); + final opened = await _repairTruncatedHeader(path); + scratch = opened.$2; + final Database src; + try { + src = await openDatabase(opened.$1, readOnly: true); + } catch (e) { + throw ImportFormatException( + 'That backup\'s database could not be opened ($e). If it came off ' + 'another phone, try exporting it again.', + ); + } + try { + return await _import(src, profile, engine, onProgress: onProgress); + } finally { + await src.close(); + } + } finally { + if (scratch != null) { + try { + await scratch.delete(recursive: true); + } catch (_) {} + } } + } + + /// A `.noopbak` exported by copying NOOP's live database file (rather than + /// through a proper backup API, e.g. `sqlite3_backup` or a WAL checkpoint) + /// can catch it mid-write: new pages are already flushed to disk, but the + /// header's page-count field (bytes 28-31, big-endian — see the SQLite file + /// format spec) was not updated before the copy ran. Every B-tree pointer + /// into one of those pages then reads as SQLITE_CORRUPT ("invalid page + /// number") from the very first page it touches, which on a real backup + /// meant losing gravity/skin-temp/step samples ENTIRELY rather than just + /// their newest rows. + /// + /// Fixed by comparing the header's page count against how many whole pages + /// the file actually holds: if there are more than the header claims, the + /// header is patched to match, on a COPY — never the original, which may be + /// the user's picked file with no [ResolvedNoopDatabase] tempDir to own it. + /// A file the header already agrees with is returned untouched, so a normal + /// backup pays no copy at all. + /// + /// This does NOT repair genuine row-level corruption from the same partial + /// write (a torn B-tree page, rather than a stale page count) — that surfaces + /// downstream as [_isCorruptPageError], and [_read] drops just the affected + /// table rather than the whole import. + static Future<(String, Directory?)> _repairTruncatedHeader( + String path, + ) async { + final f = await File(path).open(); + final int fileLen; + final List header; try { - return await _import(src, profile, engine, onProgress: onProgress); + header = await f.read(100); + fileLen = await f.length(); } finally { - await src.close(); + await f.close(); + } + if (header.length < 100 || + !const ListEquality().equals( + header.sublist(0, 16), + _kSqliteMagic, + )) { + return (path, null); // not a plain SQLite file (or too short to check) + } + final rawPageSize = (header[16] << 8) | header[17]; + // 1 means 65536 (the u16 field cannot hold it) — see the format spec. + final pageSize = rawPageSize == 1 ? 65536 : rawPageSize; + // SQLite only ever writes a power of two from 512 to 32768 (or the 1 + // encoding above). Anything else means the magic bytes matched but the + // header itself is garbage — e.g. pageSize==3 would pass a bare + // `fileLen % pageSize` check on plenty of ordinary file lengths and send + // a file that was never a real SQLite page layout through the repair + // copy, patching offset 28 against a page count that means nothing. + final validPageSize = pageSize == 65536 || + (pageSize >= 512 && pageSize <= 32768 && (pageSize & (pageSize - 1)) == 0); + if (!validPageSize || fileLen % pageSize != 0) return (path, null); + final declaredPages = + (header[28] << 24) | (header[29] << 16) | (header[30] << 8) | header[31]; + final actualPages = fileLen ~/ pageSize; + if (actualPages <= declaredPages) return (path, null); // header agrees + + final scratch = await Directory.systemTemp.createTemp('openstrap_noopfix_'); + try { + final repaired = p.join(scratch.path, 'repaired.sqlite'); + await File(path).copy(repaired); // OS-level copy, not materialised here + final raf = await File(repaired).open(mode: FileMode.append); + try { + await raf.setPosition(28); + final be = ByteData(4)..setUint32(0, actualPages, Endian.big); + await raf.writeFrom(be.buffer.asUint8List()); + } finally { + await raf.close(); + } + return (repaired, scratch); + } catch (_) { + // Nothing has been returned yet, so the caller has no handle to clean + // this up with — delete it here rather than leak a scratch dir + a + // partial copy of the user's database on every failure. + try { + await scratch.delete(recursive: true); + } catch (_) { + /* the OS reclaims the temp dir eventually */ + } + rethrow; } } @@ -154,11 +257,33 @@ class NoopBackupImporter { for (final e in columns.entries) if (e.value.length == (_kTableCols[e.key]?.length ?? -1)) e.key, }; - final span = await _span(src, readable); + + // A backup exported by copying NOOP's live database file (rather than + // through a proper backup API) can leave a scattered corrupt page + // anywhere in a table's own storage. None of these tables carry an index + // usable for a `ts` range that ALSO covers the other columns a read here + // selects (x/y/z, bpm, …), so every per-day read of an affected table + // scans the WHOLE table before it can emit a single row — the `ts` + // window narrows what's kept, not what's touched. That means one corrupt + // page anywhere in a table fails EVERY day's read of it identically, not + // just the day the bad page would chronologically belong to, so a table + // goes in here once and stays — there is no point re-running the same + // failing whole-table scan on every remaining day. + // + // Declared before the span probe (not just before the day walk below): + // `_span`'s own whole-table MIN/MAX scan can hit the same SQLITE_CORRUPT a + // per-day `_read` would, and if every table is torn this badly, `_span` + // returns null — that must still surface as "corrupted", not as "empty", + // which is the exact distinction this file exists to make. + final corrupt = {}; + final span = await _span(src, readable, corrupt); if (span == null) { - throw const ImportFormatException( - 'That NOOP backup holds no samples — there is nothing to import.', - ); + throw ImportFormatException(corrupt.isNotEmpty + ? 'That NOOP backup file is corrupted — SQLite cannot read ' + '${corrupt.join(', ')}, and those are every table this app ' + 'imports. There is nothing left to recover from this file; ' + 'export a new backup from NOOP.' + : 'That NOOP backup holds no samples — there is nothing to import.'); } final (minTs, maxTs) = span; @@ -176,41 +301,42 @@ class NoopBackupImporter { // Order within a day does not matter — [NoopIngest] rebuilds the Substrate // sorted by timestamp — but the DAYS must arrive in ascending order, since // the high-water date is what closes out and derives the previous one. - await _read(src, tables, columns, ordered, 'hrSample', from, to, (r) async { + await _read(src, tables, columns, ordered, corrupt, 'hrSample', from, to, + (r) async { final ts = _int(r['ts']), v = _int(r['bpm']); if (ts == null || v == null) return; if (await ingest.offer(ts)) ingest.hr(ts, v); }); - await _read(src, tables, columns, ordered, 'rrInterval', from, to, - (r) async { + await _read(src, tables, columns, ordered, corrupt, 'rrInterval', from, + to, (r) async { final ts = _int(r['ts']), v = _num(r['rrMs']); if (ts == null || v == null) return; if (await ingest.offer(ts)) ingest.rr(ts, v); }); - await _read(src, tables, columns, ordered, 'gravitySample', from, to, - (r) async { + await _read(src, tables, columns, ordered, corrupt, 'gravitySample', + from, to, (r) async { final ts = _int(r['ts']); if (ts == null) return; if (await ingest.offer(ts)) { ingest.gravity(ts, _num(r['x']), _num(r['y']), _num(r['z'])); } }); - await _read(src, tables, columns, ordered, 'skinTempSample', from, to, - (r) async { + await _read(src, tables, columns, ordered, corrupt, 'skinTempSample', + from, to, (r) async { final ts = _int(r['ts']); if (ts == null) return; if (await ingest.offer(ts)) ingest.skinTemp(ts, _int(r['raw'])); }); - await _read(src, tables, columns, ordered, 'spo2Sample', from, to, - (r) async { + await _read(src, tables, columns, ordered, corrupt, 'spo2Sample', from, + to, (r) async { final ts = _int(r['ts']); if (ts == null) return; if (await ingest.offer(ts)) { ingest.spo2(ts, _int(r['red']), _int(r['ir'])); } }); - await _read(src, tables, columns, ordered, 'stepSample', from, to, - (r) async { + await _read(src, tables, columns, ordered, corrupt, 'stepSample', from, + to, (r) async { final ts = _int(r['ts']), v = _int(r['counter']); if (ts == null || v == null) return; if (await ingest.offer(ts)) ingest.stepCounter(ts, v); @@ -220,13 +346,19 @@ class NoopBackupImporter { } if (ingest.rows == 0) { - throw const ImportFormatException( - 'That NOOP backup holds no samples we could read.', - ); + // Every channel we read came back corrupt (see [_isCorruptPageError]) — + // this is not "the backup is empty", it is "SQLite itself cannot read + // the file", and the two must never share a message. + throw ImportFormatException(corrupt.isNotEmpty + ? 'That NOOP backup file is corrupted — SQLite cannot read ' + '${corrupt.join(', ')}, and those are every table this app ' + 'imports. There is nothing left to recover from this file; ' + 'export a new backup from NOOP.' + : 'That NOOP backup holds no samples we could read.'); } await ingest.finish(); return NoopImportResult(ingest.days, ingest.rows, ingest.lateRows, - ingest.steps, ingest.strandedDates); + ingest.steps, ingest.strandedDates, corrupt); } /// Page one table's rows for the half-open window [from, to) into [onRow]. @@ -249,6 +381,7 @@ class NoopBackupImporter { Set tables, Map> columns, Map ordered, + Set corrupt, String table, int from, int to, @@ -259,6 +392,7 @@ class NoopBackupImporter { // channel. final cols = _kTableCols[table]!; if (!tables.contains(table)) return; + if (corrupt.contains(table)) return; // already hit a torn page, see below final usable = columns[table] ?? const []; if (usable.length < cols.length) return; final orderBy = ordered[table] ?? 'ts'; @@ -272,14 +406,25 @@ class NoopBackupImporter { num cursor = from; var firstPage = true; while (true) { - final rows = await src.query( - table, - columns: cols, - where: firstPage ? 'ts >= ? AND ts < ?' : 'ts > ? AND ts < ?', - whereArgs: [cursor, to], - orderBy: orderBy, - limit: kNoopBackupPageRows, - ); + final List> rows; + try { + rows = await src.query( + table, + columns: cols, + where: firstPage ? 'ts >= ? AND ts < ?' : 'ts > ? AND ts < ?', + whereArgs: [cursor, to], + orderBy: orderBy, + limit: kNoopBackupPageRows, + ); + } catch (e) { + if (!_isCorruptPageError(e)) rethrow; + // A whole-table scan (see the comment above [corrupt]'s declaration) + // means this can surface on page ONE of day one — there is usually + // nothing from this table to salvage, only nothing further lost by + // trying. Every OTHER channel keeps importing regardless. + corrupt.add(table); + return; + } firstPage = false; if (rows.isEmpty) return; final full = rows.length == kNoopBackupPageRows; @@ -305,15 +450,22 @@ class NoopBackupImporter { } var drained = rows.length; while (lastTs != null) { - final more = await src.query( - table, - columns: cols, - where: 'ts = ?', - whereArgs: [lastTs], - orderBy: orderBy, - limit: kNoopBackupPageRows, - offset: drained, - ); + final List> more; + try { + more = await src.query( + table, + columns: cols, + where: 'ts = ?', + whereArgs: [lastTs], + orderBy: orderBy, + limit: kNoopBackupPageRows, + offset: drained, + ); + } catch (e) { + if (!_isCorruptPageError(e)) rethrow; + corrupt.add(table); + return; + } if (more.isEmpty) break; for (final r in more) { await onRow(r); @@ -338,9 +490,24 @@ class NoopBackupImporter { } } + /// SQLite's own corruption signal (`SQLITE_CORRUPT`, result code 11 — + /// "database disk image is malformed") reached mid-scan, distinguished from + /// every other [DatabaseException] (a typo'd column, a closed database) that + /// this must NOT swallow. Matched on message text because sqflite does not + /// expose the result code uniformly across its Android/iOS/ffi backends — + /// see [DatabaseException.getResultCode]'s own comment on the same problem. + static bool _isCorruptPageError(Object e) => + e is DatabaseException && + (e.toString().contains('malformed') || + e.toString().contains('SQLITE_CORRUPT')); + /// Earliest and latest sample timestamp across every table we read, so the day /// walk covers days that carry (say) only a step counter. - static Future<(int, int)?> _span(Database src, Set tables) async { + static Future<(int, int)?> _span( + Database src, + Set tables, + Set corrupt, + ) async { int? lo, hi; for (final t in const [ 'hrSample', @@ -356,10 +523,22 @@ class NoopBackupImporter { // timestamp — one `ts = 0`, one millisecond-scaled row — would otherwise // disqualify the entire table and, if every table has one, fail the // import as "no samples" on a backup holding years of data. - final r = await src.rawQuery( - 'SELECT MIN(ts) AS lo, MAX(ts) AS hi FROM $t WHERE ts >= ? AND ts <= ?', - [_kMinPlausibleTs, _kMaxPlausibleTs], - ); + // A table torn by a live-copy export (see [_read]) can fail this + // whole-table scan before it returns a row at all. One corrupt table + // does not sink the span — every other table covers roughly the same + // 1 Hz timeline, and [_read] below marks the same table corrupt on its + // first day and drops only that channel from the import. + final List> r; + try { + r = await src.rawQuery( + 'SELECT MIN(ts) AS lo, MAX(ts) AS hi FROM $t WHERE ts >= ? AND ts <= ?', + [_kMinPlausibleTs, _kMaxPlausibleTs], + ); + } catch (e) { + if (!_isCorruptPageError(e)) rethrow; + corrupt.add(t); + continue; + } if (r.isEmpty) continue; final a = _int(r.first['lo']), b = _int(r.first['hi']); if (a == null || b == null) continue; diff --git a/lib/import/noop_import.dart b/lib/import/noop_import.dart index 7aa1d54c..c4e0567b 100644 --- a/lib/import/noop_import.dart +++ b/lib/import/noop_import.dart @@ -76,8 +76,18 @@ class NoopImportResult { /// unordered source is visible rather than quietly short. final Set strandedDates; + /// Source tables a `.noopbak` gave up on partway through — SQLite reported + /// `database disk image is malformed` reading them, which is what a backup + /// taken by copying NOOP's live database file (rather than a proper backup + /// API) leaves on its newest rows. Everything up to that point is still + /// imported; only the empty set here means the backup read cleanly. + final Set corruptTables; + NoopImportResult(this.days, this.rows, - [this.lateRows = 0, this.steps = 0, this.strandedDates = const {}]); + [this.lateRows = 0, + this.steps = 0, + this.strandedDates = const {}, + this.corruptTables = const {}]); } class NoopImporter { diff --git a/lib/ui2/onboarding/welcome.dart b/lib/ui2/onboarding/welcome.dart index 54825ef6..374b8410 100644 --- a/lib/ui2/onboarding/welcome.dart +++ b/lib/ui2/onboarding/welcome.dart @@ -47,6 +47,13 @@ class ImportOutcome { /// following day, but never derived in their own right. final int strandedDays; + /// Source tables SQLite reported as corrupt while reading a `.noopbak` + /// (unreadable pages, not a schema mismatch — see `_isCorruptPageError`). + /// Every OTHER table still imported; this only names what could not be + /// read, so a heart-rate stream that came back empty doesn't read as a + /// complete one. + final Set corruptTables; + /// Journal days written by the hand-entered CSV path (csv-reimport). Its own /// counter: those rows REPLACE the journal for the dates they name, which is /// a different promise from "a day the band measured is never overwritten", @@ -74,6 +81,7 @@ class ImportOutcome { this.skippedDays = 0, this.lateRows = 0, this.strandedDays = 0, + this.corruptTables = const {}, this.journalRows = 0, this.rejectedRows = const [], this.error, @@ -81,7 +89,8 @@ class ImportOutcome { this.readError, }); - bool get lostSomething => lateRows > 0 || strandedDays > 0; + bool get lostSomething => + lateRows > 0 || strandedDays > 0 || corruptTables.isNotEmpty; /// Nothing at all landed. A zero under a green tick is a no-op that reads as /// a success, which is the one thing an import report must never do. @@ -292,6 +301,7 @@ Future runImport( final sources = []; var days = 0, workouts = 0, skipped = 0, late = 0, stranded = 0; var journalRows = 0; + final corruptTables = {}; final rejected = []; String? rollupError; String? cryptoError; @@ -375,6 +385,7 @@ Future runImport( if (r != null) { late += r.lateRows; stranded += r.strandedDates.length; + corruptTables.addAll(r.corruptTables); } } } @@ -441,6 +452,7 @@ Future runImport( skippedDays: skipped, lateRows: late, strandedDays: stranded, + corruptTables: corruptTables, journalRows: journalRows, rejectedRows: rejected, rollupError: rollupError, @@ -709,8 +721,15 @@ class ImportReport extends StatelessWidget { l?.welcomeLateRows(o.lateRows) ?? '${o.lateRows} row${o.lateRows == 1 ? '' : 's'} arrived after ' 'their day had already been scored and closed.', + if (o.corruptTables.isNotEmpty) + '${o.corruptTables.join(', ')} could not be read — SQLite ' + 'reported the file itself as corrupted for those tables. ' + 'Every other table imported normally.', ].join(' '), - fix: l?.welcomeExportAgainInDateOrder ?? 'Export again in date order', + fix: o.corruptTables.isNotEmpty + ? (l?.actionTryAnotherFile ?? 'Try another file') + : (l?.welcomeExportAgainInDateOrder ?? + 'Export again in date order'), icon: LucideIcons.fileWarning, ), ], diff --git a/test/import_container_test.dart b/test/import_container_test.dart index 44cce1b4..e1e3fe80 100644 --- a/test/import_container_test.dart +++ b/test/import_container_test.dart @@ -10,6 +10,7 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -24,6 +25,50 @@ List _zipOf(Map members) { return ZipEncoder().encode(a); } +int _findSig(Uint8List b, List sig, [int from = 0]) { + outer: + for (var i = from; i <= b.length - sig.length; i++) { + for (var j = 0; j < sig.length; j++) { + if (b[i + j] != sig[j]) continue outer; + } + return i; + } + throw StateError('signature not found'); +} + +void _writeU32LE(Uint8List b, int offset, int value) { + b[offset] = value & 0xff; + b[offset + 1] = (value >> 8) & 0xff; + b[offset + 2] = (value >> 16) & 0xff; + b[offset + 3] = (value >> 24) & 0xff; +} + +int _readU32LE(Uint8List b, int offset) => + b[offset] | (b[offset + 1] << 8) | (b[offset + 2] << 16) | (b[offset + 3] << 24); + +/// Patch the ONE central-directory entry's declared uncompressed-size field +/// (offset +24 from its `PK\x01\x02` signature) down by [by] bytes, leaving +/// the compressed data and its CRC-32 untouched — the exact shape of the real +/// bug this guards: the size field went stale, the content did not. +Uint8List _understateCentralDirectorySize(List zipBytes, {required int by}) { + final b = Uint8List.fromList(zipBytes); + final cdr = _findSig(b, const [0x50, 0x4B, 0x01, 0x02]); + _writeU32LE(b, cdr + 24, _readU32LE(b, cdr + 24) - by); + return b; +} + +/// Corrupt the declared CRC-32 in BOTH the local file header (offset +14) and +/// the central-directory entry (offset +16), independent of which one the +/// decoder trusts — genuine damage, not merely a stale size field. +Uint8List _corruptDeclaredCrc32(List zipBytes) { + final b = Uint8List.fromList(zipBytes); + final lfh = _findSig(b, const [0x50, 0x4B, 0x03, 0x04]); + _writeU32LE(b, lfh + 14, _readU32LE(b, lfh + 14) ^ 0xFFFFFFFF); + final cdr = _findSig(b, const [0x50, 0x4B, 0x01, 0x02]); + _writeU32LE(b, cdr + 16, _readU32LE(b, cdr + 16) ^ 0xFFFFFFFF); + return b; +} + void main() { late Directory tmp; @@ -191,6 +236,50 @@ void main() { await db.dispose(); }); + // A real NOOP backup (10.5.0, measured 2026-08) unpacked to more bytes + // than its own zip declared: the central directory's uncompressed-size + // field for `noop-backup.sqlite` undercounted the true content by a few + // thousand pages, while the CRC-32 — computed over the FULL decompressed + // stream regardless of that field — still matched. `unzip -t` calls that + // file OK for the same reason. Reproduced here by encoding a normal zip + // and then patching just the declared size downward, the same shape the + // real writer's bug leaves: content and CRC untouched, only the size lie. + test('an uncompressed-size field that undercounts the real content is ' + 'not treated as damage when the CRC still matches', () async { + final content = utf8.encode('SQLite format 3 ${'z' * 5000}'); + final zipBytes = _zipOf({'noop-backup.sqlite': String.fromCharCodes(content)}); + final patched = _understateCentralDirectorySize(zipBytes, by: 37); + final path = await write('undercounted.noopbak', patched); + + final db = await resolveNoopDatabase(path); + expect(db, isNotNull); + expect(await File(db!.path).readAsBytes(), content); + await db.dispose(); + }); + + test('an uncompressed-size mismatch with a WRONG crc is still refused', + () async { + final content = utf8.encode('SQLite format 3 ${'z' * 5000}'); + final zipBytes = _zipOf({'noop-backup.sqlite': String.fromCharCodes(content)}); + // Undercount the size (as above) AND corrupt the declared CRC-32 fields + // themselves, leaving the compressed data untouched. The archive still + // decodes; what it actually produces no longer matches what it CLAIMS + // to have produced, which is real damage rather than a stale size field + // and must still be refused. + final tampered = + _corruptDeclaredCrc32(_understateCentralDirectorySize(zipBytes, by: 37)); + final path = await write('tampered.noopbak', tampered); + + await expectLater( + resolveNoopDatabase(path), + throwsA(isA().having( + (e) => e.message, + 'message', + contains('does not hold what it says'), + )), + ); + }); + test('a loose database is taken as-is and never deleted', () async { final path = await write('loose-noop.sqlite', utf8.encode('SQLite format 3 ')); final db = await resolveNoopDatabase(path); diff --git a/test/noop_backup_import_test.dart b/test/noop_backup_import_test.dart index aa592549..016071d8 100644 --- a/test/noop_backup_import_test.dart +++ b/test/noop_backup_import_test.dart @@ -9,6 +9,7 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -603,4 +604,96 @@ void main() { .having((e) => e.message, 'message', contains('no samples'))), ); }); + + // OpenStrap/edge — a real `.noopbak` off a phone (NOOP 10.5.0, WHOOP 5.0/MG) + // failed to open at all: `PRAGMA quick_check` reported "invalid page + // number" across most tables, on a zip whose own declared uncompressed size + // also undercounted its content (see import_container_test.dart for that + // half). The backup was taken by copying NOOP's live database file rather + // than through a proper backup API, which left two DIFFERENT kinds of + // damage: a stale page-count header (bytes 28-31) that undercounted how + // many pages the file actually holds, on top of genuinely torn B-tree pages + // scattered through several tables' own storage. The two tests below pin + // both, on a synthetic fixture (never the real file — see the memory note + // on personal health data in fixtures). On the real file, EVERY populated + // 1 Hz table had at least one torn page, so nothing below `dailyMetric` / + // `sleepSession` (deliberately not read — see this file's header) survived; + // the second test below still confirms the degradation is per-table, not + // total, on a fixture with exactly one bad page. + + test('a stale page-count header self-heals and imports in full', () async { + // Big enough to spread hrSample across several real pages, the same + // shape the header lie has to actually matter for. A date no other + // fixture in this file uses (they share one LocalDb across tests). + const t0 = 1786953600; // 2026-08-17 + const secs = 6000; + final dbPath = await writeNoopDb('stale_header.sqlite', t0: t0, seconds: secs); + + // Patch the header exactly the way the real bug left it: the file holds + // more whole pages than byte 28-31 claims. + final bytes = File(dbPath).readAsBytesSync(); + final pageSize = (bytes[16] << 8) | bytes[17]; + final actualPages = bytes.length ~/ pageSize; + expect(actualPages, greaterThan(4), + reason: 'fixture must span multiple pages for this to test anything'); + final understated = actualPages - 2; + final patched = Uint8List.fromList(bytes); + patched[28] = (understated >> 24) & 0xff; + patched[29] = (understated >> 16) & 0xff; + patched[30] = (understated >> 8) & 0xff; + patched[31] = understated & 0xff; + File(dbPath).writeAsBytesSync(patched); + + final bak = writeBackup('stale_header.noopbak', dbPath); + final res = await NoopImporter.importFile(bak, const Profile(), DerivationEngine()); + + // A header lie, not row damage — every row is recovered, on every table. + expect(res.rows, secs * 4 + (secs / 2).ceil()); // hr+gravity+skinTemp+step, rr every other second + expect(res.corruptTables, isEmpty); + }); + + test('a torn page loses only its own table, not the whole import', + () async { + const t0 = 1787040000; // 2026-08-18, distinct from every other fixture here + const secs = 6000; // enough rows that hrSample spans several leaf pages + final dbPath = await writeNoopDb('torn_page.sqlite', t0: t0, seconds: secs); + + // Find a real hrSample leaf page and scramble it — the same shape as a + // torn write, not merely an absent one: an all-zero page is a VALID empty + // leaf, so this has to write bytes that are not a legal btree page type. + final probe = await databaseFactory.openDatabase( + dbPath, + options: OpenDatabaseOptions(readOnly: true), + ); + final rows = await probe.rawQuery( + "SELECT pageno FROM dbstat('main') WHERE name = 'hrSample' AND pagetype = 'leaf' ORDER BY pageno", + ); + await probe.close(); + expect(rows.length, greaterThan(2), + reason: 'fixture must span multiple leaf pages for this to test anything'); + // WHICH leaf doesn't matter, and that is the point: none of these tables + // carry an index that covers the extra columns `_read` selects (bpm, x/y/z, + // …) alongside `ts`, so a day-windowed read has to scan hrSample's WHOLE + // table before it can emit a single (ordered) row. One bad page anywhere + // fails that scan for EVERY day identically — this is not "the newest + // rows are lost", it is "this whole channel is lost, every other one + // survives". + final targetPage = (rows[rows.length ~/ 2]['pageno'] as int); + + final raw = File(dbPath).readAsBytesSync(); + final pageSize = (raw[16] << 8) | raw[17]; + final patched = Uint8List.fromList(raw); + final offset = (targetPage - 1) * pageSize; + patched.fillRange(offset, offset + pageSize, 0xFF); // not a valid page type + File(dbPath).writeAsBytesSync(patched); + + final bak = writeBackup('torn_page.noopbak', dbPath); + final res = await NoopImporter.importFile(bak, const Profile(), DerivationEngine()); + + expect(res.corruptTables, {'hrSample'}); + // hrSample itself contributes NOTHING — gravity+skinTemp+step (3 * secs) + // and rr (every other second) still land in full. + expect(res.rows, secs * 3 + (secs / 2).ceil()); + expect(res.days, greaterThan(0)); + }); }