fix(stt): define and validate the direct STT audio contract (OSS-30)

README.md

@@ -212,15 +212,51 @@ await STT.load(MLXModel.GLM_ASR_Nano_4bit, {
   }
 })
 
-// Transcribe an audio buffer
+// Transcribe an audio buffer (raw mono Float32 PCM, 16 kHz by default)
 const text = await STT.transcribe(audioBuffer)
 
+// PCM at a different sample rate? Say so — it is resampled natively.
+// The language is auto-detected unless you force one.
+const spanish = await STT.transcribe(ttsBuffer, {
+  sampleRate: 24000, // e.g. audio produced by this library's TTS
+  language: 'Spanish',
+})
+
 // Or use live microphone transcription
 await STT.startListening()
 const partial = await STT.transcribeBuffer() // Get current transcript
 const final = await STT.stopListening()      // Stop and get final transcript
 ```
 
+#### Audio format
+
+`transcribe` and `transcribeStream` accept **raw native-endian mono Float32 PCM**
+only. The buffer is validated before inference:
+
+- Encoded containers (WAV, MP3 with ID3 tag, FLAC, Ogg, AIFF, CAF, MP4/M4A) are
+  rejected with an error that names the detected format. Decode to raw PCM first.
+- Byte lengths that are not a multiple of 4 (e.g. Int16 samples) are rejected.
+- The default sample rate is 16000 Hz (the model's input rate). Pass
+  `sampleRate` for other rates: values between 8000 and 48000 Hz are linearly
+  resampled to 16 kHz before inference; values outside that range are rejected.
+
+#### Microphone permission
+
+Live transcription (`startListening`) requires `NSMicrophoneUsageDescription` in
+your app's `Info.plist`. With Expo, set it in `app.json`:
+
+```json
+{
+  "expo": {
+    "ios": {
+      "infoPlist": {
+        "NSMicrophoneUsageDescription": "This app uses the microphone for speech-to-text transcription."
+      }
+    }
+  }
+}
+```
+
 ## API
 
 ### LLM
@@ -287,14 +323,27 @@ const final = await STT.stopListening()      // Stop and get final transcript
 | Method | Description |
 |--------|-------------|
 | `load(modelId: string, options?: STTLoadOptions): Promise<void>` | Load an STT model into memory |
-| `transcribe(audio: ArrayBuffer): Promise<string>` | Transcribe an audio buffer |
-| `transcribeStream(audio: ArrayBuffer, onToken: (token: string) => void): Promise<string>` | Stream transcription tokens as they're generated |
-| `startListening(): Promise<void>` | Start capturing audio from the microphone |
+| `transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise<string>` | Transcribe a raw mono Float32 PCM buffer |
+| `transcribeStream(audio: ArrayBuffer, onToken: (token: string) => void, options?: STTTranscribeOptions): Promise<string>` | Stream transcription tokens as they're generated |
+| `startListening(options?: STTListeningOptions): Promise<void>` | Start capturing audio from the microphone (requires `NSMicrophoneUsageDescription`) |
 | `transcribeBuffer(): Promise<string>` | Transcribe the current audio buffer while listening |
 | `stopListening(): Promise<string>` | Stop listening and transcribe final audio |
 | `stop(): void` | Stop the current transcription |
 | `unload(): void` | Unload the model and free memory |
 
+#### STTTranscribeOptions
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `sampleRate` | `number` | Sample rate of the provided PCM in Hz (default `16000`). Rates between `8000` and `48000` are resampled to 16 kHz natively; others are rejected. |
+| `language` | `string` | Spoken language (e.g. `'English'`, `'Spanish'`). Omitted → auto-detected. |
+
+#### STTListeningOptions
+
+| Property | Type | Description |
+|----------|------|-------------|
+| `language` | `string` | Spoken language applied to `transcribeBuffer`/`stopListening`. Omitted → auto-detected. |
+
 | Property | Description |
 |----------|-------------|
 | `isLoaded: boolean` | Whether an STT model is loaded |

docs/superpowers/specs/2026-08-11-oss-30-stt-audio-contract-design.md

@@ -0,0 +1,134 @@
+# OSS-30: Define and validate the MLX direct STT audio contract
+
+Date: 2026-08-11 · Ticket: [OSS-30](https://linear.app/henry-pl-llc/issue/OSS-30)
+
+## Problem
+
+`STT.transcribe`/`transcribeStream` treat every nonempty `ArrayBuffer` as native-endian
+mono Float32 at 16 kHz:
+
+1. **No format contract.** WAV, MP3, Int16, and odd byte lengths are silently
+   reinterpreted as Float32 samples and transcribed as garbage.
+2. **No sample-rate contract.** The library's own TTS output is 24 kHz; feeding it back
+   into STT plays it at 2/3 speed inside the model.
+3. **Forced language.** `HybridSTT` hardcodes `language: "English"` even though
+   Qwen3ASR auto-detects when `language` is `nil` and supports forcing other languages.
+4. **Missing consumer docs.** `NSMicrophoneUsageDescription` is set in the example app
+   but never documented for consumers of the library.
+
+## Approaches considered
+
+- **A. Document-only.** Narrowly document "raw PCM Float32 mono 16 kHz" and change
+  nothing. Rejected: silent corruption stays silent.
+- **B. Full audio descriptor (format/encoding/channels/rate) with decoding.** Rejected:
+  decoding WAV/MP3 containers is a codec concern; iOS already ships AVFoundation for
+  that. The library's contract stays raw PCM.
+- **C. Explicit options + validation + resampling (chosen).** The buffer stays raw
+  native-endian mono Float32. A new `STTTranscribeOptions` makes `sampleRate` and
+  `language` explicit. Both layers reject misaligned buffers and recognizable encoded
+  containers with actionable errors. Non-16-kHz PCM inside a supported range is
+  linearly resampled before inference; rates outside the range are rejected.
+
+## Design
+
+### Pure contract — `package/ios/Sources/STTAudioContract.swift`
+
+Foundation-only (no MLX import), following the `EmbeddingsBatchPlanner` pattern so the
+logic is unit-testable off-device with the `swiftc` test-script pattern.
+
+```swift
+enum STTAudioError: Error, LocalizedError, Equatable {
+  case emptyAudio
+  case misalignedAudio(byteCount: Int)
+  case encodedAudio(format: String)
+  case unsupportedSampleRate(sampleRate: Double)
+}
+
+struct STTAudioContract {
+  static let modelSampleRate: Double = 16000  // Qwen3ASR input rate, mirrored in TS
+  static let minSampleRate: Double = 8000
+  static let maxSampleRate: Double = 48000
+
+  static func detectEncodedFormat(prefix: [UInt8]) -> String?
+  static func validate(byteCount: Int, prefix: [UInt8]) throws
+  static func resolveSampleRate(_ requested: Double?) throws -> Double
+  static func resample(_ samples: [Float], from: Double, to: Double) -> [Float]
+}
+```
+
+Rules:
+
+- `byteCount == 0` → `.emptyAudio`; `byteCount % 4 != 0` → `.misalignedAudio` (raw
+  Float32 requires 4-byte frames; Int16 uploads usually trip this or the peak check in
+  consumers).
+- Container sniffing rejects unambiguous magic bytes only: `RIFF` (WAV), `ID3` (MP3),
+  `fLaC`, `OggS`, `FORM` (AIFF), `caff` (CAF), and `ftyp` at offset 4 (MP4/M4A). MP3
+  frame-sync sniffing (`0xFF 0xEx`) is deliberately omitted — those bytes occur in
+  legitimate Float32 noise.
+- `resolveSampleRate(nil)` → 16000 (the documented default). Rates must be finite and
+  within [8000, 48000]; anything else → `.unsupportedSampleRate`.
+- `resample` is linear interpolation, mono only. Identity when source == target.
+  Linear is adequate for speech ASR input; callers needing mastering-grade SRC can
+  resample upstream with AVFoundation.
+
+### `HybridSTT` wiring
+
+- `transcribe`/`transcribeStream` validate byte count + prefix via the contract before
+  binding memory, resample when `options.sampleRate != 16000`, and pass
+  `options.language` through (`nil` → Qwen3ASR auto-detect).
+- `startListening` gains `STTListeningOptions { language? }`; the stored language is
+  applied by `transcribeBuffer`/`stopListening`. The mic path already captures
+  16 kHz mono Float32, so no descriptor is needed there.
+- The previous hardcoded `"English"` default becomes auto-detect everywhere.
+
+### API — `STT.nitro.ts`
+
+```ts
+interface STTTranscribeOptions { sampleRate?: number; language?: string }
+interface STTListeningOptions { language?: string }
+
+transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise<string>
+transcribeStream(audio, onToken, options?: STTTranscribeOptions): Promise<string>
+startListening(options?: STTListeningOptions): Promise<void>
+```
+
+Nitrogen regenerated via `bun specs`.
+
+### TS-side guards — `runtime.ts`
+
+Mirror the cheap checks before crossing the bridge (Swift re-checks):
+
+- `validateSTTAudio`: ArrayBuffer type + nonempty (existing), 4-byte alignment,
+  encoded-container magic-byte rejection with the detected format named in the error.
+- `validateSTTTranscribeOptions` / `validateSTTListeningOptions`: `sampleRate` must be
+  an integer in `[STT_MIN_SAMPLE_RATE, STT_MAX_SAMPLE_RATE]`; `language` must be a
+  non-empty string when present.
+- Exported constants: `STT_SAMPLE_RATE = 16000`, `STT_MIN_SAMPLE_RATE = 8000`,
+  `STT_MAX_SAMPLE_RATE = 48000`.
+
+### README
+
+- New "Audio format" subsection under Speech-to-Text: raw PCM, Float32, mono,
+  native-endian, 16 kHz default, `sampleRate` option resamples within 8–48 kHz,
+  encoded containers rejected.
+- `language` option documented (auto-detect default).
+- Requirements section documents `NSMicrophoneUsageDescription` for live
+  transcription (with the Expo `app.json` form used by the example app).
+
+## Testing
+
+- **Swift** (`package/ios/Tests/STTAudioContractTests.swift`, run via new
+  `test:ios-stt-audio` swiftc script): empty/misaligned byte counts, each container
+  magic, non-container prefixes accepted, short prefixes, sample-rate resolution
+  (nil default, bounds, NaN/infinite/zero/negative), resample identity, 24 k→16 k
+  length and content, 8 k→16 k upsample, constant-signal preservation, empty input.
+- **TS** (`bun test`): alignment rejection, WAV/ID3/fLaC/OggS/ftyp rejection, valid
+  Float32 acceptance, sampleRate bounds and non-integer rejection, language type
+  checks, pass-through of options.
+- On-device inference (multilingual output quality) is not unit-testable in CI; the
+  contract boundary keeps that surface minimal.
+
+## Out of scope
+
+Container decoding (WAV/MP3 parsing), stereo downmix, Int16 auto-conversion, Android,
+per-chunk streaming input.

package/ios/Sources/HybridSTT.swift

@@ -15,18 +15,33 @@ class HybridSTT: HybridSTTSpec {
   private var activeTask: Task<String, Error>?
   private var loadTask: Task<Void, Error>?
   private var captureManager: AudioCaptureManager?
+  private var listeningLanguage: String?
 
   var isLoaded: Bool { model != nil }
   var isTranscribing: Bool { activeTask != nil }
   var isListening: Bool { captureManager?.isCapturing ?? false }
   var modelId: String = ""
 
-  private func arrayBufferToMLXArray(_ buffer: ArrayBuffer) -> MLXArray {
-    let count = buffer.size / MemoryLayout<Float>.size
+  /// Must stay synchronous: the JS ArrayBuffer is only valid for the duration
+  /// of the bridge call, so the copy cannot be deferred into a Task.
+  private func samplesFromArrayBuffer(
+    _ buffer: ArrayBuffer,
+    sampleRate: Double?
+  ) throws -> [Float] {
+    let byteCount = buffer.size
     let rawPtr = UnsafeRawPointer(buffer.data)
+    let prefix = [UInt8](UnsafeRawBufferPointer(start: rawPtr, count: min(byteCount, 12)))
+    try STTAudioContract.validate(byteCount: byteCount, prefix: prefix)
+    let sourceRate = try STTAudioContract.resolveSampleRate(sampleRate)
+
+    let count = byteCount / MemoryLayout<Float>.size
     let floatPtr = rawPtr.bindMemory(to: Float.self, capacity: count)
-    let floatBuffer = UnsafeBufferPointer(start: floatPtr, count: count)
-    return MLXArray(Array(floatBuffer))
+    let samples = Array(UnsafeBufferPointer(start: floatPtr, count: count))
+    return STTAudioContract.resample(
+      samples,
+      from: sourceRate,
+      to: STTAudioContract.modelSampleRate
+    )
   }
 
   func load(modelId: String, options: STTLoadOptions?) throws -> Promise<Void> {
@@ -54,15 +69,17 @@ class HybridSTT: HybridSTTSpec {
     }
   }
 
-  func transcribe(audio: ArrayBuffer) throws -> Promise<String> {
+  func transcribe(audio: ArrayBuffer, options: STTTranscribeOptions?) throws -> Promise<String> {
     guard let model else {
       throw STTError.notLoaded
     }
 
+    let samples = try samplesFromArrayBuffer(audio, sampleRate: options?.sampleRate)
+    let language = options?.language
+
     return Promise.async { [self] in
       let task = Task<String, Error> {
-        let mlxAudio = self.arrayBufferToMLXArray(audio)
-        let output = model.generate(audio: mlxAudio, language: "English")
+        let output = model.generate(audio: MLXArray(samples), language: language)
         return output.text
       }
 
@@ -75,16 +92,19 @@ class HybridSTT: HybridSTTSpec {
 
   func transcribeStream(
     audio: ArrayBuffer,
-    onToken: @escaping (_ token: String) -> Void
+    onToken: @escaping (_ token: String) -> Void,
+    options: STTTranscribeOptions?
   ) throws -> Promise<String> {
     guard let model else {
       throw STTError.notLoaded
     }
 
+    let samples = try samplesFromArrayBuffer(audio, sampleRate: options?.sampleRate)
+    let language = options?.language
+
     return Promise.async { [self] in
       let task = Task<String, Error> {
-        let mlxAudio = self.arrayBufferToMLXArray(audio)
-        let stream = model.generateStream(audio: mlxAudio, language: "English")
+        let stream = model.generateStream(audio: MLXArray(samples), language: language)
         var finalText = ""
 
         for try await event in stream {
@@ -110,14 +130,16 @@ class HybridSTT: HybridSTTSpec {
     }
   }
 
-  func startListening() throws -> Promise<Void> {
+  func startListening(options: STTListeningOptions?) throws -> Promise<Void> {
     guard model != nil else {
       throw STTError.notLoaded
     }
     guard captureManager == nil || !captureManager!.isCapturing else {
       throw STTError.alreadyListening
     }
 
+    listeningLanguage = options?.language
+
     return Promise.async { [self] in
       let manager = AudioCaptureManager()
       self.captureManager = manager
@@ -136,9 +158,10 @@ class HybridSTT: HybridSTTSpec {
       return Promise.resolved(withResult: "")
     }
 
+    let language = listeningLanguage
     return Promise.async { [self] in
       let task = Task<String, Error> {
-        let output = model.generate(audio: audio, language: "English")
+        let output = model.generate(audio: audio, language: language)
         return output.text
       }
 
@@ -161,10 +184,12 @@ class HybridSTT: HybridSTTSpec {
 
     let audio = manager.stopCapturing()
     self.captureManager = nil
+    let language = listeningLanguage
+    listeningLanguage = nil
 
     return Promise.async { [self] in
       let task = Task<String, Error> {
-        let output = model.generate(audio: audio, language: "English")
+        let output = model.generate(audio: audio, language: language)
         return output.text
       }
 
@@ -184,6 +209,7 @@ class HybridSTT: HybridSTTSpec {
       _ = manager.stopCapturing()
     }
     captureManager = nil
+    listeningLanguage = nil
   }
 
   func unload() throws {
@@ -195,6 +221,7 @@ class HybridSTT: HybridSTTSpec {
       _ = manager.stopCapturing()
     }
     captureManager = nil
+    listeningLanguage = nil
     model = nil
     modelId = ""
     Memory.clearCache()