diff --git a/README.md b/README.md index 38a80b0..076de32 100644 --- a/README.md +++ b/README.md @@ -212,15 +212,51 @@ await STT.load(MLXModel.GLM_ASR_Nano_4bit, { } }) -// Transcribe an audio buffer +// Transcribe an audio buffer (raw mono Float32 PCM, 16 kHz by default) const text = await STT.transcribe(audioBuffer) +// PCM at a different sample rate? Say so — it is resampled natively. +// The language is auto-detected unless you force one. +const spanish = await STT.transcribe(ttsBuffer, { + sampleRate: 24000, // e.g. audio produced by this library's TTS + language: 'Spanish', +}) + // Or use live microphone transcription await STT.startListening() const partial = await STT.transcribeBuffer() // Get current transcript const final = await STT.stopListening() // Stop and get final transcript ``` +#### Audio format + +`transcribe` and `transcribeStream` accept **raw native-endian mono Float32 PCM** +only. The buffer is validated before inference: + +- Encoded containers (WAV, MP3 with ID3 tag, FLAC, Ogg, AIFF, CAF, MP4/M4A) are + rejected with an error that names the detected format. Decode to raw PCM first. +- Byte lengths that are not a multiple of 4 (e.g. Int16 samples) are rejected. +- The default sample rate is 16000 Hz (the model's input rate). Pass + `sampleRate` for other rates: values between 8000 and 48000 Hz are linearly + resampled to 16 kHz before inference; values outside that range are rejected. + +#### Microphone permission + +Live transcription (`startListening`) requires `NSMicrophoneUsageDescription` in +your app's `Info.plist`. With Expo, set it in `app.json`: + +```json +{ + "expo": { + "ios": { + "infoPlist": { + "NSMicrophoneUsageDescription": "This app uses the microphone for speech-to-text transcription." + } + } + } +} +``` + ## API ### LLM @@ -287,14 +323,27 @@ const final = await STT.stopListening() // Stop and get final transcript | Method | Description | |--------|-------------| | `load(modelId: string, options?: STTLoadOptions): Promise` | Load an STT model into memory | -| `transcribe(audio: ArrayBuffer): Promise` | Transcribe an audio buffer | -| `transcribeStream(audio: ArrayBuffer, onToken: (token: string) => void): Promise` | Stream transcription tokens as they're generated | -| `startListening(): Promise` | Start capturing audio from the microphone | +| `transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise` | Transcribe a raw mono Float32 PCM buffer | +| `transcribeStream(audio: ArrayBuffer, onToken: (token: string) => void, options?: STTTranscribeOptions): Promise` | Stream transcription tokens as they're generated | +| `startListening(options?: STTListeningOptions): Promise` | Start capturing audio from the microphone (requires `NSMicrophoneUsageDescription`) | | `transcribeBuffer(): Promise` | Transcribe the current audio buffer while listening | | `stopListening(): Promise` | Stop listening and transcribe final audio | | `stop(): void` | Stop the current transcription | | `unload(): void` | Unload the model and free memory | +#### STTTranscribeOptions + +| Property | Type | Description | +|----------|------|-------------| +| `sampleRate` | `number` | Sample rate of the provided PCM in Hz (default `16000`). Rates between `8000` and `48000` are resampled to 16 kHz natively; others are rejected. | +| `language` | `string` | Spoken language (e.g. `'English'`, `'Spanish'`). Omitted → auto-detected. | + +#### STTListeningOptions + +| Property | Type | Description | +|----------|------|-------------| +| `language` | `string` | Spoken language applied to `transcribeBuffer`/`stopListening`. Omitted → auto-detected. | + | Property | Description | |----------|-------------| | `isLoaded: boolean` | Whether an STT model is loaded | diff --git a/docs/superpowers/specs/2026-08-11-oss-30-stt-audio-contract-design.md b/docs/superpowers/specs/2026-08-11-oss-30-stt-audio-contract-design.md new file mode 100644 index 0000000..5211996 --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-oss-30-stt-audio-contract-design.md @@ -0,0 +1,134 @@ +# OSS-30: Define and validate the MLX direct STT audio contract + +Date: 2026-08-11 · Ticket: [OSS-30](https://linear.app/henry-pl-llc/issue/OSS-30) + +## Problem + +`STT.transcribe`/`transcribeStream` treat every nonempty `ArrayBuffer` as native-endian +mono Float32 at 16 kHz: + +1. **No format contract.** WAV, MP3, Int16, and odd byte lengths are silently + reinterpreted as Float32 samples and transcribed as garbage. +2. **No sample-rate contract.** The library's own TTS output is 24 kHz; feeding it back + into STT plays it at 2/3 speed inside the model. +3. **Forced language.** `HybridSTT` hardcodes `language: "English"` even though + Qwen3ASR auto-detects when `language` is `nil` and supports forcing other languages. +4. **Missing consumer docs.** `NSMicrophoneUsageDescription` is set in the example app + but never documented for consumers of the library. + +## Approaches considered + +- **A. Document-only.** Narrowly document "raw PCM Float32 mono 16 kHz" and change + nothing. Rejected: silent corruption stays silent. +- **B. Full audio descriptor (format/encoding/channels/rate) with decoding.** Rejected: + decoding WAV/MP3 containers is a codec concern; iOS already ships AVFoundation for + that. The library's contract stays raw PCM. +- **C. Explicit options + validation + resampling (chosen).** The buffer stays raw + native-endian mono Float32. A new `STTTranscribeOptions` makes `sampleRate` and + `language` explicit. Both layers reject misaligned buffers and recognizable encoded + containers with actionable errors. Non-16-kHz PCM inside a supported range is + linearly resampled before inference; rates outside the range are rejected. + +## Design + +### Pure contract — `package/ios/Sources/STTAudioContract.swift` + +Foundation-only (no MLX import), following the `EmbeddingsBatchPlanner` pattern so the +logic is unit-testable off-device with the `swiftc` test-script pattern. + +```swift +enum STTAudioError: Error, LocalizedError, Equatable { + case emptyAudio + case misalignedAudio(byteCount: Int) + case encodedAudio(format: String) + case unsupportedSampleRate(sampleRate: Double) +} + +struct STTAudioContract { + static let modelSampleRate: Double = 16000 // Qwen3ASR input rate, mirrored in TS + static let minSampleRate: Double = 8000 + static let maxSampleRate: Double = 48000 + + static func detectEncodedFormat(prefix: [UInt8]) -> String? + static func validate(byteCount: Int, prefix: [UInt8]) throws + static func resolveSampleRate(_ requested: Double?) throws -> Double + static func resample(_ samples: [Float], from: Double, to: Double) -> [Float] +} +``` + +Rules: + +- `byteCount == 0` → `.emptyAudio`; `byteCount % 4 != 0` → `.misalignedAudio` (raw + Float32 requires 4-byte frames; Int16 uploads usually trip this or the peak check in + consumers). +- Container sniffing rejects unambiguous magic bytes only: `RIFF` (WAV), `ID3` (MP3), + `fLaC`, `OggS`, `FORM` (AIFF), `caff` (CAF), and `ftyp` at offset 4 (MP4/M4A). MP3 + frame-sync sniffing (`0xFF 0xEx`) is deliberately omitted — those bytes occur in + legitimate Float32 noise. +- `resolveSampleRate(nil)` → 16000 (the documented default). Rates must be finite and + within [8000, 48000]; anything else → `.unsupportedSampleRate`. +- `resample` is linear interpolation, mono only. Identity when source == target. + Linear is adequate for speech ASR input; callers needing mastering-grade SRC can + resample upstream with AVFoundation. + +### `HybridSTT` wiring + +- `transcribe`/`transcribeStream` validate byte count + prefix via the contract before + binding memory, resample when `options.sampleRate != 16000`, and pass + `options.language` through (`nil` → Qwen3ASR auto-detect). +- `startListening` gains `STTListeningOptions { language? }`; the stored language is + applied by `transcribeBuffer`/`stopListening`. The mic path already captures + 16 kHz mono Float32, so no descriptor is needed there. +- The previous hardcoded `"English"` default becomes auto-detect everywhere. + +### API — `STT.nitro.ts` + +```ts +interface STTTranscribeOptions { sampleRate?: number; language?: string } +interface STTListeningOptions { language?: string } + +transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise +transcribeStream(audio, onToken, options?: STTTranscribeOptions): Promise +startListening(options?: STTListeningOptions): Promise +``` + +Nitrogen regenerated via `bun specs`. + +### TS-side guards — `runtime.ts` + +Mirror the cheap checks before crossing the bridge (Swift re-checks): + +- `validateSTTAudio`: ArrayBuffer type + nonempty (existing), 4-byte alignment, + encoded-container magic-byte rejection with the detected format named in the error. +- `validateSTTTranscribeOptions` / `validateSTTListeningOptions`: `sampleRate` must be + an integer in `[STT_MIN_SAMPLE_RATE, STT_MAX_SAMPLE_RATE]`; `language` must be a + non-empty string when present. +- Exported constants: `STT_SAMPLE_RATE = 16000`, `STT_MIN_SAMPLE_RATE = 8000`, + `STT_MAX_SAMPLE_RATE = 48000`. + +### README + +- New "Audio format" subsection under Speech-to-Text: raw PCM, Float32, mono, + native-endian, 16 kHz default, `sampleRate` option resamples within 8–48 kHz, + encoded containers rejected. +- `language` option documented (auto-detect default). +- Requirements section documents `NSMicrophoneUsageDescription` for live + transcription (with the Expo `app.json` form used by the example app). + +## Testing + +- **Swift** (`package/ios/Tests/STTAudioContractTests.swift`, run via new + `test:ios-stt-audio` swiftc script): empty/misaligned byte counts, each container + magic, non-container prefixes accepted, short prefixes, sample-rate resolution + (nil default, bounds, NaN/infinite/zero/negative), resample identity, 24 k→16 k + length and content, 8 k→16 k upsample, constant-signal preservation, empty input. +- **TS** (`bun test`): alignment rejection, WAV/ID3/fLaC/OggS/ftyp rejection, valid + Float32 acceptance, sampleRate bounds and non-integer rejection, language type + checks, pass-through of options. +- On-device inference (multilingual output quality) is not unit-testable in CI; the + contract boundary keeps that surface minimal. + +## Out of scope + +Container decoding (WAV/MP3 parsing), stereo downmix, Int16 auto-conversion, Android, +per-chunk streaming input. diff --git a/package/ios/Sources/HybridSTT.swift b/package/ios/Sources/HybridSTT.swift index e5f35cb..f440f45 100644 --- a/package/ios/Sources/HybridSTT.swift +++ b/package/ios/Sources/HybridSTT.swift @@ -15,18 +15,33 @@ class HybridSTT: HybridSTTSpec { private var activeTask: Task? private var loadTask: Task? private var captureManager: AudioCaptureManager? + private var listeningLanguage: String? var isLoaded: Bool { model != nil } var isTranscribing: Bool { activeTask != nil } var isListening: Bool { captureManager?.isCapturing ?? false } var modelId: String = "" - private func arrayBufferToMLXArray(_ buffer: ArrayBuffer) -> MLXArray { - let count = buffer.size / MemoryLayout.size + /// Must stay synchronous: the JS ArrayBuffer is only valid for the duration + /// of the bridge call, so the copy cannot be deferred into a Task. + private func samplesFromArrayBuffer( + _ buffer: ArrayBuffer, + sampleRate: Double? + ) throws -> [Float] { + let byteCount = buffer.size let rawPtr = UnsafeRawPointer(buffer.data) + let prefix = [UInt8](UnsafeRawBufferPointer(start: rawPtr, count: min(byteCount, 12))) + try STTAudioContract.validate(byteCount: byteCount, prefix: prefix) + let sourceRate = try STTAudioContract.resolveSampleRate(sampleRate) + + let count = byteCount / MemoryLayout.size let floatPtr = rawPtr.bindMemory(to: Float.self, capacity: count) - let floatBuffer = UnsafeBufferPointer(start: floatPtr, count: count) - return MLXArray(Array(floatBuffer)) + let samples = Array(UnsafeBufferPointer(start: floatPtr, count: count)) + return STTAudioContract.resample( + samples, + from: sourceRate, + to: STTAudioContract.modelSampleRate + ) } func load(modelId: String, options: STTLoadOptions?) throws -> Promise { @@ -54,15 +69,17 @@ class HybridSTT: HybridSTTSpec { } } - func transcribe(audio: ArrayBuffer) throws -> Promise { + func transcribe(audio: ArrayBuffer, options: STTTranscribeOptions?) throws -> Promise { guard let model else { throw STTError.notLoaded } + let samples = try samplesFromArrayBuffer(audio, sampleRate: options?.sampleRate) + let language = options?.language + return Promise.async { [self] in let task = Task { - let mlxAudio = self.arrayBufferToMLXArray(audio) - let output = model.generate(audio: mlxAudio, language: "English") + let output = model.generate(audio: MLXArray(samples), language: language) return output.text } @@ -75,16 +92,19 @@ class HybridSTT: HybridSTTSpec { func transcribeStream( audio: ArrayBuffer, - onToken: @escaping (_ token: String) -> Void + onToken: @escaping (_ token: String) -> Void, + options: STTTranscribeOptions? ) throws -> Promise { guard let model else { throw STTError.notLoaded } + let samples = try samplesFromArrayBuffer(audio, sampleRate: options?.sampleRate) + let language = options?.language + return Promise.async { [self] in let task = Task { - let mlxAudio = self.arrayBufferToMLXArray(audio) - let stream = model.generateStream(audio: mlxAudio, language: "English") + let stream = model.generateStream(audio: MLXArray(samples), language: language) var finalText = "" for try await event in stream { @@ -110,7 +130,7 @@ class HybridSTT: HybridSTTSpec { } } - func startListening() throws -> Promise { + func startListening(options: STTListeningOptions?) throws -> Promise { guard model != nil else { throw STTError.notLoaded } @@ -118,6 +138,8 @@ class HybridSTT: HybridSTTSpec { throw STTError.alreadyListening } + listeningLanguage = options?.language + return Promise.async { [self] in let manager = AudioCaptureManager() self.captureManager = manager @@ -136,9 +158,10 @@ class HybridSTT: HybridSTTSpec { return Promise.resolved(withResult: "") } + let language = listeningLanguage return Promise.async { [self] in let task = Task { - let output = model.generate(audio: audio, language: "English") + let output = model.generate(audio: audio, language: language) return output.text } @@ -161,10 +184,12 @@ class HybridSTT: HybridSTTSpec { let audio = manager.stopCapturing() self.captureManager = nil + let language = listeningLanguage + listeningLanguage = nil return Promise.async { [self] in let task = Task { - let output = model.generate(audio: audio, language: "English") + let output = model.generate(audio: audio, language: language) return output.text } @@ -184,6 +209,7 @@ class HybridSTT: HybridSTTSpec { _ = manager.stopCapturing() } captureManager = nil + listeningLanguage = nil } func unload() throws { @@ -195,6 +221,7 @@ class HybridSTT: HybridSTTSpec { _ = manager.stopCapturing() } captureManager = nil + listeningLanguage = nil model = nil modelId = "" Memory.clearCache() diff --git a/package/ios/Sources/STTAudioContract.swift b/package/ios/Sources/STTAudioContract.swift new file mode 100644 index 0000000..e3be1db --- /dev/null +++ b/package/ios/Sources/STTAudioContract.swift @@ -0,0 +1,117 @@ +import Foundation + +enum STTAudioError: Error, LocalizedError, Equatable { + case emptyAudio + case misalignedAudio(byteCount: Int) + case encodedAudio(format: String) + case unsupportedSampleRate(sampleRate: Double) + + var errorDescription: String? { + switch self { + case .emptyAudio: + return "STT audio buffer is empty." + case .misalignedAudio(let byteCount): + return + "STT audio must be raw native-endian Float32 PCM; byte length \(byteCount) is not a multiple of 4." + case .encodedAudio(let format): + return + "STT audio looks like an encoded \(format) container. Decode it to raw mono Float32 PCM before calling transcribe." + case .unsupportedSampleRate(let sampleRate): + return + "STT sampleRate \(sampleRate) is unsupported. Provide a rate between \(Int(STTAudioContract.minSampleRate)) and \(Int(STTAudioContract.maxSampleRate)) Hz." + } + } + + static func == (lhs: STTAudioError, rhs: STTAudioError) -> Bool { + switch (lhs, rhs) { + case (.emptyAudio, .emptyAudio): + return true + case (.misalignedAudio(let l), .misalignedAudio(let r)): + return l == r + case (.encodedAudio(let l), .encodedAudio(let r)): + return l == r + case (.unsupportedSampleRate(let l), .unsupportedSampleRate(let r)): + // NaN never equals itself; compare bit patterns so NaN cases match in tests. + return l.bitPattern == r.bitPattern + default: + return false + } + } +} + +/// Kept free of MLX imports so `swiftc` unit tests can run off-device. +/// The TS-side guards in `runtime.ts` mirror these rules. +struct STTAudioContract { + /// Qwen3ASR input rate; mirrored in TS as `STT_SAMPLE_RATE`. + static let modelSampleRate: Double = 16000 + static let minSampleRate: Double = 8000 + static let maxSampleRate: Double = 48000 + + /// MP3 frame sync (0xFF 0xEx) is deliberately absent: those bytes occur in + /// legitimate raw Float32 sample data. + private static let signatures: [(magic: [UInt8], offset: Int, format: String)] = [ + (Array("RIFF".utf8), 0, "WAV (RIFF)"), + (Array("ID3".utf8), 0, "MP3 (ID3)"), + (Array("fLaC".utf8), 0, "FLAC"), + (Array("OggS".utf8), 0, "Ogg"), + (Array("FORM".utf8), 0, "AIFF (FORM)"), + (Array("caff".utf8), 0, "CAF"), + (Array("ftyp".utf8), 4, "MP4/M4A"), + ] + + static func detectEncodedFormat(prefix: [UInt8]) -> String? { + for signature in signatures { + let end = signature.offset + signature.magic.count + guard prefix.count >= end else { continue } + if Array(prefix[signature.offset.. 0 else { + throw STTAudioError.emptyAudio + } + guard byteCount % MemoryLayout.size == 0 else { + throw STTAudioError.misalignedAudio(byteCount: byteCount) + } + if let format = detectEncodedFormat(prefix: prefix) { + throw STTAudioError.encodedAudio(format: format) + } + } + + static func resolveSampleRate(_ requested: Double?) throws -> Double { + guard let requested else { + return modelSampleRate + } + guard requested.isFinite, requested >= minSampleRate, requested <= maxSampleRate else { + throw STTAudioError.unsupportedSampleRate(sampleRate: requested) + } + return requested + } + + /// Linear interpolation: adequate for speech input; higher-fidelity + /// resampling belongs upstream (e.g. AVFoundation). + static func resample(_ samples: [Float], from source: Double, to target: Double) -> [Float] { + guard source != target, !samples.isEmpty else { + return samples + } + + let ratio = source / target + let outputCount = max(1, Int((Double(samples.count) * target / source).rounded())) + var output = [Float]() + output.reserveCapacity(outputCount) + + for index in 0.. Bool, _ message: String) throws { + if !condition() { + throw TestFailure.failed(message) + } +} + +private func expectError( + _ expected: STTAudioError, + _ message: String, + _ body: () throws -> Void +) throws { + do { + try body() + throw TestFailure.failed("\(message): no error was thrown") + } catch let error as STTAudioError { + try expect(error == expected, "\(message): expected \(expected), received \(error)") + } +} + +private func bytes(_ ascii: String) -> [UInt8] { + Array(ascii.utf8) +} + +@main +struct STTAudioContractTests { + static func main() throws { + try expectError(.emptyAudio, "empty buffer is rejected") { + try STTAudioContract.validate(byteCount: 0, prefix: []) + } + + try expectError(.misalignedAudio(byteCount: 6), "odd byte count is rejected") { + try STTAudioContract.validate(byteCount: 6, prefix: [0, 1, 2, 3, 4, 5]) + } + try expectError(.misalignedAudio(byteCount: 2), "Int16-sized tail is rejected") { + try STTAudioContract.validate(byteCount: 2, prefix: [0, 1]) + } + + let containers: [(prefix: [UInt8], format: String)] = [ + (bytes("RIFF") + bytes("....WAVE"), "WAV (RIFF)"), + (bytes("ID3") + [3, 0, 0, 0, 0], "MP3 (ID3)"), + (bytes("fLaC") + [0, 0, 0, 0], "FLAC"), + (bytes("OggS") + [0, 0, 0, 0], "Ogg"), + (bytes("FORM") + bytes("....AIFF"), "AIFF (FORM)"), + (bytes("caff") + [0, 1, 0, 0], "CAF"), + ([0, 0, 0, 32] + bytes("ftypM4A "), "MP4/M4A"), + ] + for container in containers { + try expect( + STTAudioContract.detectEncodedFormat(prefix: container.prefix) == container.format, + "\(container.format) magic bytes are detected" + ) + try expectError( + .encodedAudio(format: container.format), + "\(container.format) container is rejected" + ) { + try STTAudioContract.validate(byteCount: 4096, prefix: container.prefix) + } + } + + let rawFloatOne: [UInt8] = [0, 0, 128, 63] + try STTAudioContract.validate(byteCount: 4, prefix: rawFloatOne) + try expect( + STTAudioContract.detectEncodedFormat(prefix: [82, 73]) == nil, + "a prefix shorter than any magic is not misdetected" + ) + + let rawSamplesResemblingMp3FrameSync: [UInt8] = [0xFF, 0xFB, 0x90, 0x00] + try STTAudioContract.validate(byteCount: 8, prefix: rawSamplesResemblingMp3FrameSync) + + let resolvedDefault = try STTAudioContract.resolveSampleRate(nil) + try expect( + resolvedDefault == STTAudioContract.modelSampleRate, + "nil sample rate resolves to the model rate" + ) + let resolved24k = try STTAudioContract.resolveSampleRate(24000) + try expect(resolved24k == 24000, "a supported sample rate resolves to itself") + let resolvedMin = try STTAudioContract.resolveSampleRate(8000) + try expect(resolvedMin == 8000, "the minimum sample rate is accepted") + let resolvedMax = try STTAudioContract.resolveSampleRate(48000) + try expect(resolvedMax == 48000, "the maximum sample rate is accepted") + + for bad in [7999.0, 48001.0, 0.0, -16000.0, Double.nan, Double.infinity] { + try expectError( + .unsupportedSampleRate(sampleRate: bad), + "sample rate \(bad) is rejected" + ) { + _ = try STTAudioContract.resolveSampleRate(bad) + } + } + + let identity: [Float] = [0.1, -0.2, 0.3] + try expect( + STTAudioContract.resample(identity, from: 16000, to: 16000) == identity, + "matching rates return the input unchanged" + ) + + try expect( + STTAudioContract.resample([], from: 24000, to: 16000).isEmpty, + "empty input resamples to empty output" + ) + + let downIn = [Float](repeating: 0.5, count: 24000) + let down = STTAudioContract.resample(downIn, from: 24000, to: 16000) + try expect(down.count == 16000, "24 kHz to 16 kHz yields 2/3 the samples, got \(down.count)") + try expect( + down.allSatisfy { abs($0 - 0.5) < 1e-6 }, + "a constant signal stays constant after downsampling" + ) + + let upIn: [Float] = [0, 1, 0, -1] + let up = STTAudioContract.resample(upIn, from: 8000, to: 16000) + try expect(up.count == 8, "8 kHz to 16 kHz doubles the samples, got \(up.count)") + try expect(abs(up[1] - 0.5) < 1e-6, "interpolated midpoints are linear, got \(up[1])") + + let ramp = (0..<240).map { Float($0) } + let rampDown = STTAudioContract.resample(ramp, from: 24000, to: 16000) + try expect(rampDown.count == 160, "ramp downsample has the expected length") + try expect( + zip(rampDown, rampDown.dropFirst()).allSatisfy { $0 < $1 }, + "a strictly increasing ramp stays strictly increasing" + ) + + try expect( + STTAudioError.misalignedAudio(byteCount: 6).errorDescription?.contains("multiple of 4") + == true, + "misaligned error explains the 4-byte requirement" + ) + try expect( + STTAudioError.encodedAudio(format: "WAV (RIFF)").errorDescription?.contains("WAV (RIFF)") + == true, + "encoded error names the detected container" + ) + try expect( + STTAudioError.unsupportedSampleRate(sampleRate: 96000).errorDescription?.contains("96000") + == true, + "sample-rate error names the offending rate" + ) + + print("STTAudioContractTests passed") + } +} diff --git a/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Bridge.hpp b/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Bridge.hpp index 3f42d9c..08431c8 100644 --- a/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Bridge.hpp +++ b/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Bridge.hpp @@ -34,8 +34,12 @@ namespace margelo::nitro::mlxreactnative { struct LLMGenerationConfig; } namespace margelo::nitro::mlxreactnative { struct LLMLoadOptions; } // Forward declaration of `LLMMessage` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct LLMMessage; } +// Forward declaration of `STTListeningOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTListeningOptions; } // Forward declaration of `STTLoadOptions` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct STTLoadOptions; } +// Forward declaration of `STTTranscribeOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTTranscribeOptions; } // Forward declaration of `StreamEventEnvelope` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct StreamEventEnvelope; } // Forward declaration of `StreamEventKind` to properly resolve imports. @@ -74,7 +78,9 @@ namespace MLXReactNative { class HybridTTSSpec_cxx; } #include "LLMGenerationConfig.hpp" #include "LLMLoadOptions.hpp" #include "LLMMessage.hpp" +#include "STTListeningOptions.hpp" #include "STTLoadOptions.hpp" +#include "STTTranscribeOptions.hpp" #include "StreamEventEnvelope.hpp" #include "StreamEventKind.hpp" #include "TTSGenerateOptions.hpp" @@ -865,6 +871,36 @@ namespace margelo::nitro::mlxreactnative::bridge::swift { return optional.value(); } + // pragma MARK: std::optional + /** + * Specialized version of `std::optional`. + */ + using std__optional_STTTranscribeOptions_ = std::optional; + inline std::optional create_std__optional_STTTranscribeOptions_(const STTTranscribeOptions& value) noexcept { + return std::optional(value); + } + inline bool has_value_std__optional_STTTranscribeOptions_(const std::optional& optional) noexcept { + return optional.has_value(); + } + inline STTTranscribeOptions get_std__optional_STTTranscribeOptions_(const std::optional& optional) noexcept { + return optional.value(); + } + + // pragma MARK: std::optional + /** + * Specialized version of `std::optional`. + */ + using std__optional_STTListeningOptions_ = std::optional; + inline std::optional create_std__optional_STTListeningOptions_(const STTListeningOptions& value) noexcept { + return std::optional(value); + } + inline bool has_value_std__optional_STTListeningOptions_(const std::optional& optional) noexcept { + return optional.has_value(); + } + inline STTListeningOptions get_std__optional_STTListeningOptions_(const std::optional& optional) noexcept { + return optional.value(); + } + // pragma MARK: std::shared_ptr /** * Specialized version of `std::shared_ptr`. diff --git a/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Umbrella.hpp b/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Umbrella.hpp index 79a6428..acd1376 100644 --- a/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Umbrella.hpp +++ b/package/nitrogen/generated/ios/MLXReactNative-Swift-Cxx-Umbrella.hpp @@ -32,8 +32,12 @@ namespace margelo::nitro::mlxreactnative { struct LLMGenerationConfig; } namespace margelo::nitro::mlxreactnative { struct LLMLoadOptions; } // Forward declaration of `LLMMessage` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct LLMMessage; } +// Forward declaration of `STTListeningOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTListeningOptions; } // Forward declaration of `STTLoadOptions` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct STTLoadOptions; } +// Forward declaration of `STTTranscribeOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTTranscribeOptions; } // Forward declaration of `StreamEventEnvelope` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct StreamEventEnvelope; } // Forward declaration of `StreamEventKind` to properly resolve imports. @@ -60,7 +64,9 @@ namespace margelo::nitro::mlxreactnative { struct ToolParameter; } #include "LLMGenerationConfig.hpp" #include "LLMLoadOptions.hpp" #include "LLMMessage.hpp" +#include "STTListeningOptions.hpp" #include "STTLoadOptions.hpp" +#include "STTTranscribeOptions.hpp" #include "StreamEventEnvelope.hpp" #include "StreamEventKind.hpp" #include "TTSGenerateOptions.hpp" diff --git a/package/nitrogen/generated/ios/c++/HybridSTTSpecSwift.hpp b/package/nitrogen/generated/ios/c++/HybridSTTSpecSwift.hpp index 5bda631..a3d52ee 100644 --- a/package/nitrogen/generated/ios/c++/HybridSTTSpecSwift.hpp +++ b/package/nitrogen/generated/ios/c++/HybridSTTSpecSwift.hpp @@ -16,6 +16,10 @@ namespace MLXReactNative { class HybridSTTSpec_cxx; } namespace margelo::nitro::mlxreactnative { struct STTLoadOptions; } // Forward declaration of `ArrayBufferHolder` to properly resolve imports. namespace NitroModules { class ArrayBufferHolder; } +// Forward declaration of `STTTranscribeOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTTranscribeOptions; } +// Forward declaration of `STTListeningOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTListeningOptions; } #include #include @@ -24,6 +28,8 @@ namespace NitroModules { class ArrayBufferHolder; } #include #include #include +#include "STTTranscribeOptions.hpp" +#include "STTListeningOptions.hpp" #include "MLXReactNative-Swift-Cxx-Umbrella.hpp" @@ -95,24 +101,24 @@ namespace margelo::nitro::mlxreactnative { auto __value = std::move(__result.value()); return __value; } - inline std::shared_ptr> transcribe(const std::shared_ptr& audio) override { - auto __result = _swiftPart.transcribe(ArrayBufferHolder(audio)); + inline std::shared_ptr> transcribe(const std::shared_ptr& audio, const std::optional& options) override { + auto __result = _swiftPart.transcribe(ArrayBufferHolder(audio), options); if (__result.hasError()) [[unlikely]] { std::rethrow_exception(__result.error()); } auto __value = std::move(__result.value()); return __value; } - inline std::shared_ptr> transcribeStream(const std::shared_ptr& audio, const std::function& onToken) override { - auto __result = _swiftPart.transcribeStream(ArrayBufferHolder(audio), onToken); + inline std::shared_ptr> transcribeStream(const std::shared_ptr& audio, const std::function& onToken, const std::optional& options) override { + auto __result = _swiftPart.transcribeStream(ArrayBufferHolder(audio), onToken, options); if (__result.hasError()) [[unlikely]] { std::rethrow_exception(__result.error()); } auto __value = std::move(__result.value()); return __value; } - inline std::shared_ptr> startListening() override { - auto __result = _swiftPart.startListening(); + inline std::shared_ptr> startListening(const std::optional& options) override { + auto __result = _swiftPart.startListening(options); if (__result.hasError()) [[unlikely]] { std::rethrow_exception(__result.error()); } diff --git a/package/nitrogen/generated/ios/swift/HybridSTTSpec.swift b/package/nitrogen/generated/ios/swift/HybridSTTSpec.swift index c29c29d..5e3f521 100644 --- a/package/nitrogen/generated/ios/swift/HybridSTTSpec.swift +++ b/package/nitrogen/generated/ios/swift/HybridSTTSpec.swift @@ -17,9 +17,9 @@ public protocol HybridSTTSpec_protocol: HybridObject { // Methods func load(modelId: String, options: STTLoadOptions?) throws -> Promise - func transcribe(audio: ArrayBuffer) throws -> Promise - func transcribeStream(audio: ArrayBuffer, onToken: @escaping (_ token: String) -> Void) throws -> Promise - func startListening() throws -> Promise + func transcribe(audio: ArrayBuffer, options: STTTranscribeOptions?) throws -> Promise + func transcribeStream(audio: ArrayBuffer, onToken: @escaping (_ token: String) -> Void, options: STTTranscribeOptions?) throws -> Promise + func startListening(options: STTListeningOptions?) throws -> Promise func transcribeBuffer() throws -> Promise func stopListening() throws -> Promise func stop() throws -> Void diff --git a/package/nitrogen/generated/ios/swift/HybridSTTSpec_cxx.swift b/package/nitrogen/generated/ios/swift/HybridSTTSpec_cxx.swift index eaf6a55..3296cc6 100644 --- a/package/nitrogen/generated/ios/swift/HybridSTTSpec_cxx.swift +++ b/package/nitrogen/generated/ios/swift/HybridSTTSpec_cxx.swift @@ -170,9 +170,9 @@ open class HybridSTTSpec_cxx { } @inline(__always) - public final func transcribe(audio: ArrayBuffer) -> bridge.Result_std__shared_ptr_Promise_std__string___ { + public final func transcribe(audio: ArrayBuffer, options: bridge.std__optional_STTTranscribeOptions_) -> bridge.Result_std__shared_ptr_Promise_std__string___ { do { - let __result = try self.__implementation.transcribe(audio: audio) + let __result = try self.__implementation.transcribe(audio: audio, options: options.value) let __resultCpp = { () -> bridge.std__shared_ptr_Promise_std__string__ in let __promise = bridge.create_std__shared_ptr_Promise_std__string__() let __promiseHolder = bridge.wrap_std__shared_ptr_Promise_std__string__(__promise) @@ -189,14 +189,14 @@ open class HybridSTTSpec_cxx { } @inline(__always) - public final func transcribeStream(audio: ArrayBuffer, onToken: bridge.Func_void_std__string) -> bridge.Result_std__shared_ptr_Promise_std__string___ { + public final func transcribeStream(audio: ArrayBuffer, onToken: bridge.Func_void_std__string, options: bridge.std__optional_STTTranscribeOptions_) -> bridge.Result_std__shared_ptr_Promise_std__string___ { do { let __result = try self.__implementation.transcribeStream(audio: audio, onToken: { () -> (String) -> Void in let __wrappedFunction = bridge.wrap_Func_void_std__string(onToken) return { (__token: String) -> Void in __wrappedFunction.call(std.string(__token)) } - }()) + }(), options: options.value) let __resultCpp = { () -> bridge.std__shared_ptr_Promise_std__string__ in let __promise = bridge.create_std__shared_ptr_Promise_std__string__() let __promiseHolder = bridge.wrap_std__shared_ptr_Promise_std__string__(__promise) @@ -213,9 +213,9 @@ open class HybridSTTSpec_cxx { } @inline(__always) - public final func startListening() -> bridge.Result_std__shared_ptr_Promise_void___ { + public final func startListening(options: bridge.std__optional_STTListeningOptions_) -> bridge.Result_std__shared_ptr_Promise_void___ { do { - let __result = try self.__implementation.startListening() + let __result = try self.__implementation.startListening(options: options.value) let __resultCpp = { () -> bridge.std__shared_ptr_Promise_void__ in let __promise = bridge.create_std__shared_ptr_Promise_void__() let __promiseHolder = bridge.wrap_std__shared_ptr_Promise_void__(__promise) diff --git a/package/nitrogen/generated/ios/swift/STTListeningOptions.swift b/package/nitrogen/generated/ios/swift/STTListeningOptions.swift new file mode 100644 index 0000000..9e186a0 --- /dev/null +++ b/package/nitrogen/generated/ios/swift/STTListeningOptions.swift @@ -0,0 +1,42 @@ +/// +/// STTListeningOptions.swift +/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. +/// https://github.com/mrousavy/nitro +/// Copyright © Marc Rousavy @ Margelo +/// + +import NitroModules + +/** + * Represents an instance of `STTListeningOptions`, backed by a C++ struct. + */ +public typealias STTListeningOptions = margelo.nitro.mlxreactnative.STTListeningOptions + +public extension STTListeningOptions { + private typealias bridge = margelo.nitro.mlxreactnative.bridge.swift + + /** + * Create a new instance of `STTListeningOptions`. + */ + init(language: String?) { + self.init({ () -> bridge.std__optional_std__string_ in + if let __unwrappedValue = language { + return bridge.create_std__optional_std__string_(std.string(__unwrappedValue)) + } else { + return .init() + } + }()) + } + + @inline(__always) + var language: String? { + return { () -> String? in + if bridge.has_value_std__optional_std__string_(self.__language) { + let __unwrapped = bridge.get_std__optional_std__string_(self.__language) + return String(__unwrapped) + } else { + return nil + } + }() + } +} diff --git a/package/nitrogen/generated/ios/swift/STTTranscribeOptions.swift b/package/nitrogen/generated/ios/swift/STTTranscribeOptions.swift new file mode 100644 index 0000000..91f7a4d --- /dev/null +++ b/package/nitrogen/generated/ios/swift/STTTranscribeOptions.swift @@ -0,0 +1,60 @@ +/// +/// STTTranscribeOptions.swift +/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. +/// https://github.com/mrousavy/nitro +/// Copyright © Marc Rousavy @ Margelo +/// + +import NitroModules + +/** + * Represents an instance of `STTTranscribeOptions`, backed by a C++ struct. + */ +public typealias STTTranscribeOptions = margelo.nitro.mlxreactnative.STTTranscribeOptions + +public extension STTTranscribeOptions { + private typealias bridge = margelo.nitro.mlxreactnative.bridge.swift + + /** + * Create a new instance of `STTTranscribeOptions`. + */ + init(sampleRate: Double?, language: String?) { + self.init({ () -> bridge.std__optional_double_ in + if let __unwrappedValue = sampleRate { + return bridge.create_std__optional_double_(__unwrappedValue) + } else { + return .init() + } + }(), { () -> bridge.std__optional_std__string_ in + if let __unwrappedValue = language { + return bridge.create_std__optional_std__string_(std.string(__unwrappedValue)) + } else { + return .init() + } + }()) + } + + @inline(__always) + var sampleRate: Double? { + return { () -> Double? in + if bridge.has_value_std__optional_double_(self.__sampleRate) { + let __unwrapped = bridge.get_std__optional_double_(self.__sampleRate) + return __unwrapped + } else { + return nil + } + }() + } + + @inline(__always) + var language: String? { + return { () -> String? in + if bridge.has_value_std__optional_std__string_(self.__language) { + let __unwrapped = bridge.get_std__optional_std__string_(self.__language) + return String(__unwrapped) + } else { + return nil + } + }() + } +} diff --git a/package/nitrogen/generated/shared/c++/HybridSTTSpec.hpp b/package/nitrogen/generated/shared/c++/HybridSTTSpec.hpp index 99c7c0a..84a860a 100644 --- a/package/nitrogen/generated/shared/c++/HybridSTTSpec.hpp +++ b/package/nitrogen/generated/shared/c++/HybridSTTSpec.hpp @@ -15,13 +15,19 @@ // Forward declaration of `STTLoadOptions` to properly resolve imports. namespace margelo::nitro::mlxreactnative { struct STTLoadOptions; } +// Forward declaration of `STTTranscribeOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTTranscribeOptions; } +// Forward declaration of `STTListeningOptions` to properly resolve imports. +namespace margelo::nitro::mlxreactnative { struct STTListeningOptions; } #include #include #include "STTLoadOptions.hpp" #include #include +#include "STTTranscribeOptions.hpp" #include +#include "STTListeningOptions.hpp" namespace margelo::nitro::mlxreactnative { @@ -58,9 +64,9 @@ namespace margelo::nitro::mlxreactnative { public: // Methods virtual std::shared_ptr> load(const std::string& modelId, const std::optional& options) = 0; - virtual std::shared_ptr> transcribe(const std::shared_ptr& audio) = 0; - virtual std::shared_ptr> transcribeStream(const std::shared_ptr& audio, const std::function& onToken) = 0; - virtual std::shared_ptr> startListening() = 0; + virtual std::shared_ptr> transcribe(const std::shared_ptr& audio, const std::optional& options) = 0; + virtual std::shared_ptr> transcribeStream(const std::shared_ptr& audio, const std::function& onToken, const std::optional& options) = 0; + virtual std::shared_ptr> startListening(const std::optional& options) = 0; virtual std::shared_ptr> transcribeBuffer() = 0; virtual std::shared_ptr> stopListening() = 0; virtual void stop() = 0; diff --git a/package/nitrogen/generated/shared/c++/STTListeningOptions.hpp b/package/nitrogen/generated/shared/c++/STTListeningOptions.hpp new file mode 100644 index 0000000..df60c5a --- /dev/null +++ b/package/nitrogen/generated/shared/c++/STTListeningOptions.hpp @@ -0,0 +1,84 @@ +/// +/// STTListeningOptions.hpp +/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. +/// https://github.com/mrousavy/nitro +/// Copyright © Marc Rousavy @ Margelo +/// + +#pragma once + +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif + + + +#include +#include + +namespace margelo::nitro::mlxreactnative { + + /** + * A struct which can be represented as a JavaScript object (STTListeningOptions). + */ + struct STTListeningOptions final { + public: + std::optional language SWIFT_PRIVATE; + + public: + STTListeningOptions() = default; + explicit STTListeningOptions(std::optional language): language(language) {} + + public: + friend bool operator==(const STTListeningOptions& lhs, const STTListeningOptions& rhs) = default; + }; + +} // namespace margelo::nitro::mlxreactnative + +namespace margelo::nitro { + + // C++ STTListeningOptions <> JS STTListeningOptions (object) + template <> + struct JSIConverter final { + static inline margelo::nitro::mlxreactnative::STTListeningOptions fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) { + jsi::Object obj = arg.asObject(runtime); + return margelo::nitro::mlxreactnative::STTListeningOptions( + JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "language"))) + ); + } + static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::mlxreactnative::STTListeningOptions& arg) { + jsi::Object obj(runtime); + obj.setProperty(runtime, PropNameIDCache::get(runtime, "language"), JSIConverter>::toJSI(runtime, arg.language)); + return obj; + } + static inline bool canConvert(jsi::Runtime& runtime, const jsi::Value& value) { + if (!value.isObject()) { + return false; + } + jsi::Object obj = value.getObject(runtime); + if (!nitro::isPlainObject(runtime, obj)) { + return false; + } + if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "language")))) return false; + return true; + } + }; + +} // namespace margelo::nitro diff --git a/package/nitrogen/generated/shared/c++/STTTranscribeOptions.hpp b/package/nitrogen/generated/shared/c++/STTTranscribeOptions.hpp new file mode 100644 index 0000000..c3a71a5 --- /dev/null +++ b/package/nitrogen/generated/shared/c++/STTTranscribeOptions.hpp @@ -0,0 +1,88 @@ +/// +/// STTTranscribeOptions.hpp +/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE. +/// https://github.com/mrousavy/nitro +/// Copyright © Marc Rousavy @ Margelo +/// + +#pragma once + +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif +#if __has_include() +#include +#else +#error NitroModules cannot be found! Are you sure you installed NitroModules properly? +#endif + + + +#include +#include + +namespace margelo::nitro::mlxreactnative { + + /** + * A struct which can be represented as a JavaScript object (STTTranscribeOptions). + */ + struct STTTranscribeOptions final { + public: + std::optional sampleRate SWIFT_PRIVATE; + std::optional language SWIFT_PRIVATE; + + public: + STTTranscribeOptions() = default; + explicit STTTranscribeOptions(std::optional sampleRate, std::optional language): sampleRate(sampleRate), language(language) {} + + public: + friend bool operator==(const STTTranscribeOptions& lhs, const STTTranscribeOptions& rhs) = default; + }; + +} // namespace margelo::nitro::mlxreactnative + +namespace margelo::nitro { + + // C++ STTTranscribeOptions <> JS STTTranscribeOptions (object) + template <> + struct JSIConverter final { + static inline margelo::nitro::mlxreactnative::STTTranscribeOptions fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) { + jsi::Object obj = arg.asObject(runtime); + return margelo::nitro::mlxreactnative::STTTranscribeOptions( + JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "sampleRate"))), + JSIConverter>::fromJSI(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "language"))) + ); + } + static inline jsi::Value toJSI(jsi::Runtime& runtime, const margelo::nitro::mlxreactnative::STTTranscribeOptions& arg) { + jsi::Object obj(runtime); + obj.setProperty(runtime, PropNameIDCache::get(runtime, "sampleRate"), JSIConverter>::toJSI(runtime, arg.sampleRate)); + obj.setProperty(runtime, PropNameIDCache::get(runtime, "language"), JSIConverter>::toJSI(runtime, arg.language)); + return obj; + } + static inline bool canConvert(jsi::Runtime& runtime, const jsi::Value& value) { + if (!value.isObject()) { + return false; + } + jsi::Object obj = value.getObject(runtime); + if (!nitro::isPlainObject(runtime, obj)) { + return false; + } + if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "sampleRate")))) return false; + if (!JSIConverter>::canConvert(runtime, obj.getProperty(runtime, PropNameIDCache::get(runtime, "language")))) return false; + return true; + } + }; + +} // namespace margelo::nitro diff --git a/package/package.json b/package/package.json index 4f85175..4222a94 100644 --- a/package/package.json +++ b/package/package.json @@ -16,6 +16,7 @@ "test:ios-generation-task": "swiftc ios/Sources/LLMError.swift ios/Sources/GenerationTaskController.swift ios/Tests/GenerationTaskControllerTests.swift -o /tmp/GenerationTaskControllerTests && /tmp/GenerationTaskControllerTests", "test:ios-tts-speed": "swiftc ios/Sources/TTSSpeed.swift ios/Tests/TTSSpeedTests.swift -o /tmp/TTSSpeedTests && /tmp/TTSSpeedTests", "test:ios-embeddings-planner": "swiftc ios/Sources/EmbeddingsBatchPlanner.swift ios/Tests/EmbeddingsBatchPlannerTests.swift -o /tmp/EmbeddingsBatchPlannerTests && /tmp/EmbeddingsBatchPlannerTests", + "test:ios-stt-audio": "swiftc ios/Sources/STTAudioContract.swift ios/Tests/STTAudioContractTests.swift -o /tmp/STTAudioContractTests && /tmp/STTAudioContractTests", "clean": "rm -rf android/build node_modules/**/android/build lib android/.cxx node_modules/**/android/.cxx", "release": "release-it", "specs": "bun typecheck && nitrogen --logLevel=\\\"debug\\\" && bun run build", diff --git a/package/src/index.ts b/package/src/index.ts index 90ca17d..bd50417 100755 --- a/package/src/index.ts +++ b/package/src/index.ts @@ -64,7 +64,9 @@ export type { export type { ModelManager as ModelManagerSpec } from './specs/ModelManager.nitro' export type { STT as STTSpec, + STTListeningOptions, STTLoadOptions, + STTTranscribeOptions, STTTranscriptionInfo, } from './specs/STT.nitro' export type { diff --git a/package/src/runtime.test.ts b/package/src/runtime.test.ts index 73ba2e7..f368c71 100644 --- a/package/src/runtime.test.ts +++ b/package/src/runtime.test.ts @@ -6,12 +6,18 @@ import { createSafeCallback, EMBEDDINGS_MAX_BATCH_SIZE, mapStreamEventEnvelope, + STT_MAX_SAMPLE_RATE, + STT_MIN_SAMPLE_RATE, + STT_SAMPLE_RATE, safeJsonParse, TTS_MAX_SPEED, TTS_MIN_SPEED, validateEmbeddingsBatch, validateEmbeddingsEmbedOptions, validateLLMLoadOptions, + validateSTTAudio, + validateSTTListeningOptions, + validateSTTTranscribeOptions, validateTTSGenerateOptions, } from './runtime' @@ -252,3 +258,100 @@ describe('embeddings guards', () => { expect(() => validateEmbeddingsBatch(['ok', ' '])).toThrow('texts[1]') }) }) + +function audioWithHeader(ascii: string, offset = 0, byteLength = 64): ArrayBuffer { + const buffer = new ArrayBuffer(byteLength) + const view = new Uint8Array(buffer) + for (let i = 0; i < ascii.length; i++) { + view[offset + i] = ascii.charCodeAt(i) + } + return buffer +} + +describe('STT audio contract', () => { + it('exposes the model sample-rate constants', () => { + expect(STT_SAMPLE_RATE).toBe(16000) + expect(STT_MIN_SAMPLE_RATE).toBe(8000) + expect(STT_MAX_SAMPLE_RATE).toBe(48000) + }) + + it('accepts a raw Float32 buffer', () => { + const buffer = new Float32Array([0, 0.5, -0.5, 1]).buffer + expect(validateSTTAudio(buffer, 'STT audio')).toBe(buffer) + }) + + it('rejects byte lengths that are not a multiple of 4', () => { + expect(() => validateSTTAudio(new ArrayBuffer(6), 'STT audio')).toThrow( + 'multiple of 4', + ) + }) + + it('rejects recognizable encoded containers', () => { + expect(() => validateSTTAudio(audioWithHeader('RIFF'), 'STT audio')).toThrow( + 'WAV (RIFF)', + ) + expect(() => validateSTTAudio(audioWithHeader('ID3'), 'STT audio')).toThrow( + 'MP3 (ID3)', + ) + expect(() => validateSTTAudio(audioWithHeader('fLaC'), 'STT audio')).toThrow('FLAC') + expect(() => validateSTTAudio(audioWithHeader('OggS'), 'STT audio')).toThrow('Ogg') + expect(() => validateSTTAudio(audioWithHeader('FORM'), 'STT audio')).toThrow('AIFF') + expect(() => validateSTTAudio(audioWithHeader('caff'), 'STT audio')).toThrow('CAF') + expect(() => validateSTTAudio(audioWithHeader('ftyp', 4), 'STT audio')).toThrow( + 'MP4/M4A', + ) + }) + + it('does not misdetect MP3 frame sync in raw sample data', () => { + const buffer = new ArrayBuffer(8) + new Uint8Array(buffer).set([0xff, 0xfb, 0x90, 0x00, 0, 0, 0, 0]) + expect(validateSTTAudio(buffer, 'STT audio')).toBe(buffer) + }) + + it('rejects empty and non-ArrayBuffer audio', () => { + expect(() => validateSTTAudio(new ArrayBuffer(0), 'STT audio')).toThrow( + 'must not be empty', + ) + expect(() => validateSTTAudio('audio', 'STT audio')).toThrow('must be an ArrayBuffer') + }) + + it('accepts transcribe options within the contract', () => { + expect(validateSTTTranscribeOptions(undefined)).toBeUndefined() + expect(validateSTTTranscribeOptions({})).toEqual({}) + expect(validateSTTTranscribeOptions({ sampleRate: 24000 })).toEqual({ + sampleRate: 24000, + }) + expect(validateSTTTranscribeOptions({ language: 'Spanish' })).toEqual({ + language: 'Spanish', + }) + }) + + it('rejects out-of-range or non-integer sample rates', () => { + expect(() => + validateSTTTranscribeOptions({ sampleRate: STT_MIN_SAMPLE_RATE - 1 }), + ).toThrow('between 8000 and 48000') + expect(() => + validateSTTTranscribeOptions({ sampleRate: STT_MAX_SAMPLE_RATE + 1 }), + ).toThrow('between 8000 and 48000') + expect(() => validateSTTTranscribeOptions({ sampleRate: Number.NaN })).toThrow( + 'integer', + ) + expect(() => validateSTTTranscribeOptions({ sampleRate: 44100.5 })).toThrow('integer') + }) + + it('rejects empty languages', () => { + expect(() => validateSTTTranscribeOptions({ language: ' ' })).toThrow( + 'non-empty string', + ) + expect(() => validateSTTListeningOptions({ language: '' })).toThrow( + 'non-empty string', + ) + }) + + it('accepts listening options', () => { + expect(validateSTTListeningOptions(undefined)).toBeUndefined() + expect(validateSTTListeningOptions({ language: 'French' })).toEqual({ + language: 'French', + }) + }) +}) diff --git a/package/src/runtime.ts b/package/src/runtime.ts index c68c07e..54b8722 100644 --- a/package/src/runtime.ts +++ b/package/src/runtime.ts @@ -9,7 +9,11 @@ import type { StreamEventEnvelope, ToolDefinition, } from './specs/LLM.nitro' -import type { STTLoadOptions } from './specs/STT.nitro' +import type { + STTListeningOptions, + STTLoadOptions, + STTTranscribeOptions, +} from './specs/STT.nitro' import type { TTSGenerateOptions, TTSLoadOptions } from './specs/TTS.nitro' const ERROR_PREFIX = '[react-native-nitro-mlx]' @@ -139,6 +143,104 @@ export function validateSTTLoadOptions( return validateLoadOptions(options, 'STT') } +/** Mirrors `STTAudioContract.modelSampleRate` on the native side. */ +export const STT_SAMPLE_RATE = 16000 +export const STT_MIN_SAMPLE_RATE = 8000 +export const STT_MAX_SAMPLE_RATE = 48000 + +/** + * Mirrors `STTAudioContract.signatures` on the native side. MP3 frame sync + * (0xFF 0xEx) is deliberately absent — those bytes occur in raw Float32 data. + */ +const STT_ENCODED_SIGNATURES: ReadonlyArray<{ + magic: string + offset: number + format: string +}> = [ + { magic: 'RIFF', offset: 0, format: 'WAV (RIFF)' }, + { magic: 'ID3', offset: 0, format: 'MP3 (ID3)' }, + { magic: 'fLaC', offset: 0, format: 'FLAC' }, + { magic: 'OggS', offset: 0, format: 'Ogg' }, + { magic: 'FORM', offset: 0, format: 'AIFF (FORM)' }, + { magic: 'caff', offset: 0, format: 'CAF' }, + { magic: 'ftyp', offset: 4, format: 'MP4/M4A' }, +] + +function detectEncodedAudioFormat(buffer: ArrayBuffer): string | null { + const view = new Uint8Array(buffer, 0, Math.min(buffer.byteLength, 12)) + for (const { magic, offset, format } of STT_ENCODED_SIGNATURES) { + if (view.length < offset + magic.length) { + continue + } + let matches = true + for (let i = 0; i < magic.length; i++) { + if (view[offset + i] !== magic.charCodeAt(i)) { + matches = false + break + } + } + if (matches) { + return format + } + } + return null +} + +export function validateSTTAudio(value: unknown, name: string): ArrayBuffer { + const buffer = assertArrayBuffer(value, name) + if (buffer.byteLength % 4 !== 0) { + throw new TypeError( + `${ERROR_PREFIX} ${name} must be raw native-endian mono Float32 PCM; byte length ${buffer.byteLength} is not a multiple of 4.`, + ) + } + const format = detectEncodedAudioFormat(buffer) + if (format) { + throw new TypeError( + `${ERROR_PREFIX} ${name} looks like an encoded ${format} container. Decode it to raw mono Float32 PCM before transcribing.`, + ) + } + return buffer +} + +function validateSTTLanguage(language: unknown): void { + if (language !== undefined) { + assertNonEmptyString(language, 'STT language') + } +} + +export function validateSTTTranscribeOptions( + options?: STTTranscribeOptions, +): STTTranscribeOptions | undefined { + if (!options) { + return undefined + } + if (options.sampleRate !== undefined) { + if (!Number.isInteger(options.sampleRate)) { + throw new TypeError(`${ERROR_PREFIX} STT sampleRate must be an integer in Hz.`) + } + if ( + options.sampleRate < STT_MIN_SAMPLE_RATE || + options.sampleRate > STT_MAX_SAMPLE_RATE + ) { + throw new RangeError( + `${ERROR_PREFIX} STT sampleRate must be between ${STT_MIN_SAMPLE_RATE} and ${STT_MAX_SAMPLE_RATE} Hz.`, + ) + } + } + validateSTTLanguage(options.language) + return options +} + +export function validateSTTListeningOptions( + options?: STTListeningOptions, +): STTListeningOptions | undefined { + if (!options) { + return undefined + } + validateSTTLanguage(options.language) + return options +} + export function validateEmbeddingsLoadOptions( options?: EmbeddingsLoadOptions, ): EmbeddingsLoadOptions | undefined { diff --git a/package/src/specs/STT.nitro.ts b/package/src/specs/STT.nitro.ts index 0dfe711..578e218 100644 --- a/package/src/specs/STT.nitro.ts +++ b/package/src/specs/STT.nitro.ts @@ -4,6 +4,25 @@ export interface STTLoadOptions { onProgress?: (progress: number) => void } +/** Options for transcribing raw native-endian mono Float32 PCM buffers. */ +export interface STTTranscribeOptions { + /** + * Sample rate of the PCM in Hz (default 16000). Rates within 8000–48000 are + * resampled natively before inference; others are rejected. + */ + sampleRate?: number + /** Spoken language (e.g. `'Spanish'`). Omit to auto-detect. */ + language?: string +} + +export interface STTListeningOptions { + /** + * Spoken language applied to `transcribeBuffer`/`stopListening` results. + * Omit to auto-detect. + */ + language?: string +} + export interface STTTranscriptionInfo { promptTokens: number generationTokens: number @@ -20,10 +39,14 @@ export interface STT extends HybridObject<{ ios: 'swift' }> { load(modelId: string, options?: STTLoadOptions): Promise - transcribe(audio: ArrayBuffer): Promise - transcribeStream(audio: ArrayBuffer, onToken: (token: string) => void): Promise + transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise + transcribeStream( + audio: ArrayBuffer, + onToken: (token: string) => void, + options?: STTTranscribeOptions, + ): Promise - startListening(): Promise + startListening(options?: STTListeningOptions): Promise transcribeBuffer(): Promise stopListening(): Promise diff --git a/package/src/stt.ts b/package/src/stt.ts index 77ac3a0..047ea64 100644 --- a/package/src/stt.ts +++ b/package/src/stt.ts @@ -1,11 +1,18 @@ import { NitroModules } from 'react-native-nitro-modules' import { - assertArrayBuffer, assertNonEmptyString, createSafeCallback, + validateSTTAudio, + validateSTTListeningOptions, validateSTTLoadOptions, + validateSTTTranscribeOptions, } from './runtime' -import type { STTLoadOptions, STT as STTSpec } from './specs/STT.nitro' +import type { + STTListeningOptions, + STTLoadOptions, + STT as STTSpec, + STTTranscribeOptions, +} from './specs/STT.nitro' let instance: STTSpec | null = null @@ -27,22 +34,27 @@ export const STT = { ) }, - transcribe(audio: ArrayBuffer): Promise { - return getInstance().transcribe(assertArrayBuffer(audio, 'STT audio')) + transcribe(audio: ArrayBuffer, options?: STTTranscribeOptions): Promise { + return getInstance().transcribe( + validateSTTAudio(audio, 'STT audio'), + validateSTTTranscribeOptions(options), + ) }, transcribeStream( audio: ArrayBuffer, onToken: (token: string) => void, + options?: STTTranscribeOptions, ): Promise { return getInstance().transcribeStream( - assertArrayBuffer(audio, 'STT audio'), + validateSTTAudio(audio, 'STT audio'), createSafeCallback('STT.transcribeStream onToken', onToken) ?? (() => {}), + validateSTTTranscribeOptions(options), ) }, - startListening(): Promise { - return getInstance().startListening() + startListening(options?: STTListeningOptions): Promise { + return getInstance().startListening(validateSTTListeningOptions(options)) }, transcribeBuffer(): Promise {