diff --git a/.github/workflows/release-apk.yml b/.github/workflows/release-apk.yml index 61eef99..e35c209 100644 --- a/.github/workflows/release-apk.yml +++ b/.github/workflows/release-apk.yml @@ -2,6 +2,8 @@ name: Build Release APK on: push: + branches: + - "codex/moonshine-vocabulary-beta" tags: - "v*" workflow_dispatch: @@ -11,7 +13,7 @@ permissions: jobs: build: - name: Build dev debug APK + name: Build APK runs-on: ubuntu-latest steps: @@ -20,6 +22,18 @@ jobs: with: submodules: recursive + - name: Configure release signing + env: + RELEASE_KEYSTORE_BASE64: ${{ secrets.RELEASE_KEYSTORE_BASE64 }} + RELEASE_KEY_ALIAS: ${{ secrets.RELEASE_KEY_ALIAS }} + RELEASE_KEY_PASSWORD: ${{ secrets.RELEASE_KEY_PASSWORD }} + RELEASE_STORE_PASSWORD: ${{ secrets.RELEASE_STORE_PASSWORD }} + run: | + test -n "$RELEASE_KEYSTORE_BASE64" + printf '%s' "$RELEASE_KEYSTORE_BASE64" | base64 --decode > release-signing.jks + printf 'storeFile=release-signing.jks\nkeyAlias=%s\nkeyPassword=%s\nstorePassword=%s\n' \ + "$RELEASE_KEY_ALIAS" "$RELEASE_KEY_PASSWORD" "$RELEASE_STORE_PASSWORD" > keystore.properties + - name: Set up JDK uses: actions/setup-java@v4 with: @@ -43,34 +57,38 @@ jobs: run: cargo install cargo-ndk - name: Build APK - run: ./gradlew :app:assembleStandaloneRelease + run: | + ./gradlew :app:assembleStandaloneRelease + echo "APK_DIR=app/build/outputs/apk/standalone/release" >> "$GITHUB_ENV" + echo "APK_NAME=futo-voice-input-moonshine" >> "$GITHUB_ENV" - name: Rename APK run: | mkdir -p release VERSION="${GITHUB_REF_NAME:-manual}" - APK_PATH="$(find app/build/outputs/apk/standalone/release -maxdepth 1 -type f -name '*.apk' | head -n 1)" + VERSION="${VERSION//\//-}" + APK_PATH="$(find "$APK_DIR" -maxdepth 1 -type f -name '*.apk' | head -n 1)" if [ -z "$APK_PATH" ]; then - echo "No APK found in app/build/outputs/apk/standalone/release" + echo "No APK found in $APK_DIR" exit 1 fi - cp "$APK_PATH" "release/futo-voice-input-parakeet-${VERSION}.apk" + cp "$APK_PATH" "release/${APK_NAME}-${VERSION}.apk" - name: Upload APK artifact uses: actions/upload-artifact@v4 with: - name: futo-voice-input-parakeet-apk + name: futo-voice-input-moonshine-apk path: release/*.apk - name: Create GitHub Release if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v2 with: - name: FUTO Voice Input Parakeet ${{ github.ref_name }} + name: FUTO Voice Input Moonshine ${{ github.ref_name }} body: | - Runtime-download Parakeet release of the FUTO Voice Input fork, now branded as FUTO Voice Input Parakeet. + Offline Moonshine streaming release of the FUTO Voice Input fork, with Parakeet and Whisper retained as selectable backends. - This APK no longer packages the Parakeet ONNX model files directly. The app downloads the Parakeet model on first use or from the Model Options screen, which keeps the APK much smaller while preserving offline transcription after the model is installed. + Models download on first use or from Model Options, keeping the APK smaller while preserving offline transcription after installation. The Parakeet integration and repository changes in this fork were built with AI assistance using Codex (GPT-5). files: release/*.apk diff --git a/AGENTS.md b/AGENTS.md index f833bc7..35951cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ This is a personal fork of FUTO Voice Input. Upstreams: - Original GitLab: `https://gitlab.futo.org/keyboard/voiceinput` - Public fork: `https://github.com/Today20092/futo_with_parakeet` -The goal is to preserve FUTO Voice Input's Android IME/activity UX while making NVIDIA Parakeet TDT 0.6B V3 via ONNX Runtime the default offline recognizer. Legacy FUTO Whisper/GGML remains selectable from Model Options. +The goal is to preserve FUTO Voice Input's Android IME/activity UX while making Moonshine v2 Small Streaming the default offline recognizer. NVIDIA Parakeet TDT 0.6B V3 and legacy FUTO Whisper/GGML remain selectable from Model Options. ## High-Level Architecture @@ -28,7 +28,7 @@ Primary runtime flow: 2. Both wrap `RecognizerView`, which owns the Compose recognition UI and forwards lifecycle events. 3. `RecognizerView` delegates recording/model work to `AudioRecognizer`. 4. `AudioRecognizer` records 16 kHz mono PCM with `AudioRecord`, applies WebRTC VAD, buffers float samples, then selects a `SpeechBackend`. -5. Default backend is `ParakeetBackend`; optional backend is `WhisperGGMLBackend`. +5. Default backend is `MoonshineBackend`; optional backends are `ParakeetBackend` and `WhisperGGMLBackend`. 6. `ParakeetBackend` calls `ParakeetNative`, which loads `c++_shared`, `onnxruntime`, and `parakeet_voiceinput`. 7. Rust JNI functions in `parakeet-native/src/lib.rs` call `parakeet-native/src/engine.rs`, which uses `transcribe-rs` and ONNX Runtime. @@ -40,8 +40,8 @@ Primary runtime flow: - `app/src/main/java/org/futo/voiceinput/RecognizerView.kt`: recognition UI state machine, sounds, permission/model prompts, result dispatch hooks. - `app/src/main/java/org/futo/voiceinput/RecognizeActivity.kt`: speech recognizer activity entry point. - `app/src/main/java/org/futo/voiceinput/VoiceInputMethodService.kt`: IME entry point, commits final/partial text to the active input connection, switches back on cancel. -- `app/src/main/java/org/futo/voiceinput/settings/Settings.kt`: DataStore keys. `SPEECH_BACKEND` defaults to `parakeet`. -- `app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt`: Model Options UI for Parakeet vs Whisper/GGML. +- `app/src/main/java/org/futo/voiceinput/settings/Settings.kt`: DataStore keys. `SPEECH_BACKEND` defaults to `moonshine`. +- `app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt`: Model Options UI for Moonshine, Parakeet, and Whisper/GGML. - `app/src/main/java/org/futo/voiceinput/downloader/DownloadActivity.kt`: shared model downloader. Supports explicit file URLs/hashes for Parakeet and legacy FUTO model names for Whisper. - `app/src/main/java/org/futo/voiceinput/parakeet/ParakeetModel.kt`: Parakeet file list, Hugging Face URLs, download marker, hash verification, model download intent. - `app/src/main/java/org/futo/voiceinput/parakeet/ParakeetEngineManager.kt`: shared/warm Parakeet backend, idle unload timeout. @@ -116,3 +116,17 @@ Generated APK names are customized in `app/build.gradle`, but the GitHub workflo - Do not remove legacy Whisper/GGML paths unless explicitly requested; this fork intentionally keeps them as selectable fallback. - Be careful with generated/native outputs under `app/src/main/jniLibs/`; they are produced by Gradle/Rust build tasks. - This repo may have local uncommitted changes. Check `git status --short` before editing and avoid reverting user changes. + +## Agent skills + +### Issue tracker + +Issues are tracked in GitHub Issues; external PRs are not a triage surface. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Use the canonical labels `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`. + +### Domain docs + +This is a single-context repository. See `docs/agents/domain.md`. diff --git a/README.md b/README.md index 06d3ff8..57b4fed 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,60 @@ -# FUTO Voice Input Parakeet +# FUTO Voice Input Moonshine -This is a personal fork of FUTO Voice Input, renamed as FUTO Voice Input Parakeet, that keeps the FUTO voice keyboard experience and uses NVIDIA Parakeet TDT 0.6B V3 through ONNX Runtime by default. The legacy FUTO Whisper/GGML backend can also be selected from Model Options. +This personal fork keeps the FUTO voice keyboard experience and uses Moonshine v2 Small Streaming as its default offline recognizer. NVIDIA Parakeet TDT 0.6B V3 and the legacy FUTO Whisper/GGML backend remain selectable from Model Options. This fork's Parakeet integration and repository changes were built with AI assistance using Codex (GPT-5). -The goal is straightforward: keep the FUTO UI, recording flow, dark theme support, VAD silence stopping, microphone animation, and keyboard switch-back behavior, while using Parakeet as the main recognizer because it is fast and accurate for English speech. +The goal is straightforward: keep the FUTO UI and recording flow while adding responsive streaming transcription and personal vocabulary corrections. ## What Changed -- FUTO Voice Input Parakeet remains the base Android app and UI. -- Parakeet is the default speech backend. -- The legacy Whisper/GGML recognition path can be selected as an optional backend. -- A Kotlin backend layer calls a Rust JNI library for Parakeet. -- The Rust library loads Parakeet ONNX models through `transcribe-rs` and ONNX Runtime. -- The Model Options screen lets you choose Parakeet or Whisper/GGML. +- Moonshine v2 Small Streaming is the default backend and emits live partial transcripts. +- Parakeet and legacy Whisper/GGML remain selectable backends. +- Batch and streaming recognizers share backend-neutral Kotlin contracts. +- Personal vocabulary entries correct partial and final transcripts; use `heard => preferred` for explicit aliases. +- The stable app uses the distinct `org.futo.voiceinput.moonshine` package ID. - Only the selected backend's model files are required before voice input starts. ## Screenshots ### Model Options -Model Options screen with Parakeet selected +Model Options screen -Parakeet is selected as the default backend, with Whisper/GGML still available as a fallback. +Moonshine is selected by default, with Parakeet and Whisper/GGML available as fallbacks. ## Active Model The default backend is: ```text -Unified Parakeet TDT 0.6B V3 +Moonshine v2 Small Streaming English ``` -The app downloads the ONNX model assets from: +The app downloads the quantized model assets from: ```text -https://huggingface.co/istupakov/parakeet-tdt-0.6b-v3-onnx +https://download.moonshine.ai/model/small-streaming-en/quantized/ ``` -By default, the Parakeet model files are not packaged into the APK. This keeps the APK much smaller than the first bundled-model release. +Model files are downloaded on first use rather than packaged into the APK. After installing the APK, the model is downloaded by the app: -1. Open FUTO Voice Input Parakeet Settings. +1. Open FUTO Voice Input Moonshine Settings. 2. Tap **Model**. -3. Tap **Unified Parakeet TDT 0.6B V3** under **Parakeet Model**. +3. Select **Moonshine v2 Small Streaming**. 4. Confirm the download. -If you try to use voice input before downloading the model, the app shows a download prompt before it starts recording. Once the download finishes, the app loads Parakeet from local app storage and transcription runs offline. +If you try voice input before downloading the model, the app prompts for the download. Transcription runs offline after installation. Downloaded model files are stored in app-private storage: ```text -filesDir/parakeet-tdt-0.6b-v3-int8/ +filesDir/moonshine-small-streaming-en/ ``` -If you select **Whisper/GGML** in Model Options, the app uses the legacy FUTO English and multilingual GGML model settings and downloader. Those GGML files are also stored in app-private storage and run offline after download. Parakeet downloads from Hugging Face, while Whisper/GGML models download through the existing FUTO model downloader. +Parakeet and Whisper/GGML models are also stored in app-private storage and run offline after download. ## Building Locally @@ -131,8 +130,8 @@ You can also run the workflow manually from the Actions tab. Manual runs upload - First supported ABI is `arm64-v8a`. - This is intended for sideloading and personal testing. -- The normal APK does not include the Parakeet ONNX model files. The model downloads at runtime from Hugging Face. -- Parakeet currently returns a final transcript after recording stops; live partial transcripts are not implemented. +- The normal APK does not include speech model files. +- Moonshine emits live partial transcripts; Parakeet currently returns only a final transcript. - The app requires network access to download the selected backend's model the first time, then transcription runs offline. ## Attribution And License diff --git a/app/build.gradle b/app/build.gradle index 643b40e..f6cc068 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -34,10 +34,10 @@ android { defaultConfig { applicationId "org.futo.voiceinput" - minSdk 24 + minSdk 26 targetSdk 35 - versionCode 35 - versionName "1.3.7-8-parakeet-branding" + versionCode 36 + versionName "1.4.0" buildConfigField "boolean", "BUNDLE_PARAKEET_MODEL", bundleParakeetModel.toString() testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" @@ -122,6 +122,7 @@ android { jniLibs { useLegacyPackaging false keepDebugSymbols += '**/*.so' + pickFirsts += '**/libonnxruntime.so' } } @@ -133,6 +134,8 @@ android { standalone { dimension "version" + applicationIdSuffix ".moonshine" + versionNameSuffix "-moonshine" } fDroid { @@ -229,7 +232,7 @@ android { android.applicationVariants.all { variant -> variant.outputs.all { - def appSlug = "futo-voice-input-parakeet" + def appSlug = "futo-voice-input-moonshine" def flavor = variant.flavorName ?: "unflavored" def version = variant.versionName.replaceAll(/[^A-Za-z0-9._-]/, "-") def buildType = variant.buildType.name @@ -271,6 +274,7 @@ dependencies { implementation 'com.squareup.okhttp3:okhttp:4.11.0' implementation 'com.microsoft.onnxruntime:onnxruntime-android:1.22.0' + implementation 'ai.moonshine:moonshine-voice:0.0.68' implementation 'ch.acra:acra-http:5.11.1' implementation 'ch.acra:acra-dialog:5.11.1' diff --git a/app/src/dev/res/values/strings.xml b/app/src/dev/res/values/strings.xml index 6440eee..fb83222 100644 --- a/app/src/dev/res/values/strings.xml +++ b/app/src/dev/res/values/strings.xml @@ -1,4 +1,4 @@ - FUTO Voice Input Parakeet [Dev] - FUTO Voice Input Parakeet Method [Dev] + FUTO Voice Input Moonshine Beta + FUTO Voice Input Moonshine Method [Beta] diff --git a/app/src/main/java/org/futo/voiceinput/AudioRecognizer.kt b/app/src/main/java/org/futo/voiceinput/AudioRecognizer.kt index 7cb00ef..269e2fb 100644 --- a/app/src/main/java/org/futo/voiceinput/AudioRecognizer.kt +++ b/app/src/main/java/org/futo/voiceinput/AudioRecognizer.kt @@ -27,18 +27,22 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.coroutines.yield import org.futo.voiceinput.ml.RunState +import org.futo.voiceinput.moonshine.MoonshineBackend +import org.futo.voiceinput.moonshine.isMoonshineModelDownloaded import org.futo.voiceinput.settings.ENABLE_30S_LIMIT import org.futo.voiceinput.settings.IS_VAD_ENABLED import org.futo.voiceinput.settings.MANUAL_STOP_DRAIN_MS import org.futo.voiceinput.settings.PARAKEET_KEEP_WARM import org.futo.voiceinput.settings.PARAKEET_KEEP_WARM_TIMEOUT_MS import org.futo.voiceinput.settings.PARAKEET_USE_VAD +import org.futo.voiceinput.settings.PERSONAL_DICTIONARY import org.futo.voiceinput.settings.SPEECH_BACKEND import org.futo.voiceinput.settings.SpeechBackendType import org.futo.voiceinput.settings.getSetting import org.futo.voiceinput.parakeet.ParakeetBackend import org.futo.voiceinput.parakeet.ParakeetEngineManager -import org.futo.voiceinput.parakeet.SpeechBackend +import org.futo.voiceinput.backend.SpeechBackend +import org.futo.voiceinput.backend.StreamingSpeechBackend import org.futo.voiceinput.parakeet.isParakeetModelDownloaded import org.futo.voiceinput.settings.toSpeechBackendType import java.nio.FloatBuffer @@ -84,6 +88,7 @@ abstract class AudioRecognizer { private var recorderJob: Job? = null private var modelJob: Job? = null private var loadModelJob: Job? = null + private var personalVocabulary = "" private var canExpandSpace = true private fun expandSpaceIfAllowed(): Boolean { @@ -109,6 +114,7 @@ abstract class AudioRecognizer { protected abstract fun loading() protected abstract fun needParakeetModelDownload() + protected abstract fun needMoonshineModelDownload() protected abstract fun needWhisperModelDownload(models: List) protected abstract fun needPermission() protected abstract fun permissionRejected() @@ -175,6 +181,7 @@ abstract class AudioRecognizer { stopAndReleaseRecorder() recorderJob?.cancel() modelJob?.cancel() + loadModelJob?.cancel() isRecording = false floatSamples.clear() @@ -183,6 +190,8 @@ abstract class AudioRecognizer { lifecycleScope.launch { modelJob?.join() + loadModelJob?.join() + loadModelJob = null if (shouldForceCloseParakeet) { ParakeetEngineManager.forceClose() } @@ -241,6 +250,10 @@ abstract class AudioRecognizer { val backendType = context.getSetting(SPEECH_BACKEND).toSpeechBackendType() backend = when (backendType) { SpeechBackendType.Parakeet -> ParakeetEngineManager.acquire(context) + SpeechBackendType.Moonshine -> { + ParakeetEngineManager.forceClose() + MoonshineBackend().also { it.load(context) } + } SpeechBackendType.WhisperGGML -> { ParakeetEngineManager.forceClose() WhisperGGMLBackend( @@ -296,6 +309,7 @@ abstract class AudioRecognizer { lifecycleScope.launch { val backendType = context.getSetting(SPEECH_BACKEND).toSpeechBackendType() + personalVocabulary = context.getSetting(PERSONAL_DICTIONARY) when (backendType) { SpeechBackendType.Parakeet -> { if (!context.isParakeetModelDownloaded(verifyHashes = true)) { @@ -303,6 +317,14 @@ abstract class AudioRecognizer { return@launch } } + SpeechBackendType.Moonshine -> { + if (!context.isMoonshineModelDownloaded()) { + needMoonshineModelDownload() + return@launch + } + loadModel() + loadModelJob?.join() + } SpeechBackendType.WhisperGGML -> { val requiredModels = context.selectedWhisperModelsForCurrentSettings(forcedLanguage) if (requiredModels.any { context.modelNeedsDownloading(it) }) { @@ -376,6 +398,14 @@ abstract class AudioRecognizer { focusAudio() isRecording = true + (backend as? StreamingSpeechBackend)?.startStreaming { result -> + lifecycleScope.launch { + withContext(Dispatchers.Main) { + partialResult(PersonalVocabulary.apply(result, personalVocabulary)) + } + } + } + val canMicBeBlocked = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { (context.getSystemService(SensorPrivacyManager::class.java) as SensorPrivacyManager).supportsSensorToggle( SensorPrivacyManager.Sensors.MICROPHONE @@ -567,9 +597,14 @@ abstract class AudioRecognizer { return false } + val streamingBackend = backend as? StreamingSpeechBackend + val streamingChunk = if (streamingBackend != null) FloatArray(nRead) else null for(i in 0 until nRead) { - floatSamples.put(samples[i].toFloat() / Short.MAX_VALUE.toFloat()) + val sample = samples[i].toFloat() / Short.MAX_VALUE.toFloat() + floatSamples.put(sample) + streamingChunk?.set(i, sample) } + streamingChunk?.let { streamingBackend?.acceptAudio(it) } return true } @@ -612,6 +647,7 @@ abstract class AudioRecognizer { repeat(silenceSamples) { floatSamples.put(0.0f) } + (backend as? StreamingSpeechBackend)?.acceptAudio(FloatArray(silenceSamples)) } private suspend fun runModel(){ @@ -646,7 +682,9 @@ abstract class AudioRecognizer { yield() val text = try { - backend!!.transcribe(floatArray) + val rawText = (backend as? StreamingSpeechBackend)?.finishStreaming() + ?: backend!!.transcribe(floatArray) + PersonalVocabulary.apply(rawText, personalVocabulary) } catch(e: OutOfMemoryError) { decodingStatus(RunState.OOMError) val backendType = context.getSetting(SPEECH_BACKEND).toSpeechBackendType() diff --git a/app/src/main/java/org/futo/voiceinput/PersonalVocabulary.kt b/app/src/main/java/org/futo/voiceinput/PersonalVocabulary.kt new file mode 100644 index 0000000..6fac232 --- /dev/null +++ b/app/src/main/java/org/futo/voiceinput/PersonalVocabulary.kt @@ -0,0 +1,87 @@ +package org.futo.voiceinput + +object PersonalVocabulary { + fun apply(text: String, vocabulary: String): String { + if (text.isBlank() || vocabulary.isBlank()) return text + + val entries = vocabulary.split(ENTRY_SEPARATOR) + .map(String::trim) + .filter(String::isNotEmpty) + var corrected = text + + entries.mapNotNull { entry -> + entry.split("=>", limit = 2).takeIf { it.size == 2 } + ?.map(String::trim) + ?.takeIf { it.all(String::isNotEmpty) } + }.forEach { (heard, preferred) -> + corrected = Regex( + "(?i)(?") } + .sortedByDescending { normalize(it).length } + .forEach { preferred -> corrected = correctTerm(corrected, preferred) } + + return corrected + } + + private fun correctTerm(text: String, preferred: String): String { + val preferredWords = WORD.findAll(preferred).toList() + if (preferredWords.isEmpty()) return text + + val words = WORD.findAll(text).toList() + val preferredWidth = preferredWords.size + val wanted = normalize(preferred) + val maxDistance = when { + wanted.length >= 10 -> 2 + wanted.length >= 5 -> 1 + else -> 0 + } + + val replacements = mutableListOf() + var index = 0 + while (index < words.size) { + val widths = (preferredWidth + 1 downTo maxOf(1, preferredWidth - 1)) + val matchedWidth = widths.firstOrNull { width -> + if (index + width > words.size) return@firstOrNull false + val first = words[index] + val last = words[index + width - 1] + val candidate = normalize(text.substring(first.range.first, last.range.last + 1)) + editDistance(candidate, wanted) <= maxDistance + } + if (matchedWidth == null) { + index++ + continue + } + replacements += words[index].range.first..words[index + matchedWidth - 1].range.last + index += matchedWidth + } + return replacements.asReversed().fold(text) { result, range -> + result.replaceRange(range, preferred) + } + } + + private fun normalize(value: String): String = + value.lowercase().filter(Char::isLetterOrDigit) + + private fun editDistance(left: String, right: String): Int { + var previous = IntArray(right.length + 1) { it } + left.forEachIndexed { leftIndex, leftChar -> + val current = IntArray(right.length + 1) + current[0] = leftIndex + 1 + right.forEachIndexed { rightIndex, rightChar -> + current[rightIndex + 1] = minOf( + current[rightIndex] + 1, + previous[rightIndex + 1] + 1, + previous[rightIndex] + if (leftChar == rightChar) 0 else 1 + ) + } + previous = current + } + return previous.last() + } + + private val WORD = Regex("[\\p{L}\\p{N}][\\p{L}\\p{N}'’-]*") + private val ENTRY_SEPARATOR = Regex("[,\\r\\n]+") +} diff --git a/app/src/main/java/org/futo/voiceinput/RecognizeActivity.kt b/app/src/main/java/org/futo/voiceinput/RecognizeActivity.kt index f0a06ee..6e4f923 100644 --- a/app/src/main/java/org/futo/voiceinput/RecognizeActivity.kt +++ b/app/src/main/java/org/futo/voiceinput/RecognizeActivity.kt @@ -38,6 +38,7 @@ import androidx.lifecycle.LifecycleCoroutineScope import androidx.lifecycle.lifecycleScope import org.futo.voiceinput.migration.scheduleModelMigrationJob import org.futo.voiceinput.parakeet.parakeetModelDownloadIntent +import org.futo.voiceinput.moonshine.moonshineModelDownloadIntent import org.futo.voiceinput.settings.pages.ConditionalUnpaidNoticeInVoiceInputWindow import org.futo.voiceinput.theme.UixThemeAuto import org.futo.voiceinput.updates.scheduleUpdateCheckingJob @@ -156,6 +157,10 @@ class RecognizeActivity : ComponentActivity() { this@RecognizeActivity.requestParakeetModelDownload() } + override fun requestMoonshineModelDownload() { + this@RecognizeActivity.requestMoonshineModelDownload() + } + override fun requestWhisperModelDownload(models: List) { this@RecognizeActivity.requestWhisperModelDownload(models) } @@ -214,6 +219,10 @@ class RecognizeActivity : ComponentActivity() { modelDownload.launch(parakeetModelDownloadIntent()) } + private fun requestMoonshineModelDownload() { + modelDownload.launch(moonshineModelDownloadIntent()) + } + private fun requestWhisperModelDownload(models: List) { modelDownload.launch(modelDownloadIntent(models)) } diff --git a/app/src/main/java/org/futo/voiceinput/RecognizerView.kt b/app/src/main/java/org/futo/voiceinput/RecognizerView.kt index 6214bb3..dc2f839 100644 --- a/app/src/main/java/org/futo/voiceinput/RecognizerView.kt +++ b/app/src/main/java/org/futo/voiceinput/RecognizerView.kt @@ -311,6 +311,7 @@ abstract class RecognizerView { abstract fun sendPartialResult(result: String): Boolean abstract fun requestPermission() abstract fun requestParakeetModelDownload() + abstract fun requestMoonshineModelDownload() abstract fun requestWhisperModelDownload(models: List) abstract fun decodingStarted() @@ -448,6 +449,22 @@ abstract class RecognizerView { } } + override fun needMoonshineModelDownload() { + setContent { + this@RecognizerView.Window( + onClose = { cancelRecognizer() }, + onFinish = { requestMoonshineModelDownload() }, + onPauseVAD = { }, + allowClick = false + ) { + RecognizeModelDownloadRequired( + body = context.getString(R.string.moonshine_download_required_body), + onDownload = { requestMoonshineModelDownload() } + ) + } + } + } + override fun needWhisperModelDownload(models: List) { setContent { this@RecognizerView.Window( diff --git a/app/src/main/java/org/futo/voiceinput/VoiceInputMethodService.kt b/app/src/main/java/org/futo/voiceinput/VoiceInputMethodService.kt index ab47271..bdf07f8 100644 --- a/app/src/main/java/org/futo/voiceinput/VoiceInputMethodService.kt +++ b/app/src/main/java/org/futo/voiceinput/VoiceInputMethodService.kt @@ -63,6 +63,7 @@ import androidx.savedstate.findViewTreeSavedStateRegistryOwner import androidx.savedstate.setViewTreeSavedStateRegistryOwner import org.futo.voiceinput.migration.scheduleModelMigrationJob import org.futo.voiceinput.parakeet.startParakeetModelDownloadActivity +import org.futo.voiceinput.moonshine.startMoonshineModelDownloadActivity import org.futo.voiceinput.settings.pages.ConditionalUnpaidNoticeInVoiceInputWindow import org.futo.voiceinput.theme.UixThemeAuto import org.futo.voiceinput.updates.scheduleUpdateCheckingJob @@ -285,6 +286,11 @@ class VoiceInputMethodService : InputMethodService(), LifecycleOwner, ViewModelS onCancel() } + override fun requestMoonshineModelDownload() { + this@VoiceInputMethodService.startMoonshineModelDownloadActivity() + onCancel() + } + override fun requestWhisperModelDownload(models: List) { this@VoiceInputMethodService.startModelDownloadActivity(models) onCancel() diff --git a/app/src/main/java/org/futo/voiceinput/WhisperGGMLBackend.kt b/app/src/main/java/org/futo/voiceinput/WhisperGGMLBackend.kt index f1d3d35..90e56a2 100644 --- a/app/src/main/java/org/futo/voiceinput/WhisperGGMLBackend.kt +++ b/app/src/main/java/org/futo/voiceinput/WhisperGGMLBackend.kt @@ -1,10 +1,10 @@ package org.futo.voiceinput import android.content.Context +import org.futo.voiceinput.backend.SpeechBackend import org.futo.voiceinput.ggml.DecodingMode import org.futo.voiceinput.ml.RunState import org.futo.voiceinput.ml.WhisperModelWrapper -import org.futo.voiceinput.parakeet.SpeechBackend import org.futo.voiceinput.settings.BEAM_SEARCH import org.futo.voiceinput.settings.DISALLOW_SYMBOLS import org.futo.voiceinput.settings.ENGLISH_MODEL_INDEX diff --git a/app/src/main/java/org/futo/voiceinput/backend/SpeechBackend.kt b/app/src/main/java/org/futo/voiceinput/backend/SpeechBackend.kt new file mode 100644 index 0000000..27ce799 --- /dev/null +++ b/app/src/main/java/org/futo/voiceinput/backend/SpeechBackend.kt @@ -0,0 +1,15 @@ +package org.futo.voiceinput.backend + +import android.content.Context + +interface SpeechBackend { + suspend fun load(context: Context) + suspend fun transcribe(samples: FloatArray): String + suspend fun close() +} + +interface StreamingSpeechBackend : SpeechBackend { + fun startStreaming(onPartial: (String) -> Unit) + fun acceptAudio(samples: FloatArray) + suspend fun finishStreaming(): String +} diff --git a/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineBackend.kt b/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineBackend.kt new file mode 100644 index 0000000..9240b03 --- /dev/null +++ b/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineBackend.kt @@ -0,0 +1,106 @@ +package org.futo.voiceinput.moonshine + +import ai.moonshine.voice.JNI +import ai.moonshine.voice.Transcriber +import ai.moonshine.voice.TranscriptEvent +import ai.moonshine.voice.TranscriptEventListener +import android.content.Context +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.futo.voiceinput.backend.StreamingSpeechBackend + +class MoonshineBackend : StreamingSpeechBackend { + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private var transcriber: Transcriber? = null + private var audio: Channel? = null + private var worker: Job? = null + private var completedText = "" + private var currentText = "" + + override suspend fun load(context: Context) = withContext(Dispatchers.IO) { + transcriber = Transcriber().apply { + loadFromFiles( + context.applicationContext.moonshineModelDir().absolutePath, + JNI.MOONSHINE_MODEL_ARCH_SMALL_STREAMING + ) + } + } + + override suspend fun transcribe(samples: FloatArray): String = withContext(Dispatchers.Default) { + transcriberOrThrow().transcribeWithoutStreaming(samples, 16_000).lines + .mapNotNull { it.text } + .joinToString(" ") + .trim() + } + + override fun startStreaming(onPartial: (String) -> Unit) { + completedText = "" + currentText = "" + val transcriber = transcriberOrThrow() + transcriber.removeAllListeners() + transcriber.addListener { event -> + event.accept(object : TranscriptEventListener() { + override fun onLineTextChanged(event: TranscriptEvent.LineTextChanged) { + currentText = event.line.text.orEmpty() + onPartial(currentTranscript()) + } + + override fun onLineCompleted(event: TranscriptEvent.LineCompleted) { + completedText = listOf(completedText, event.line.text.orEmpty()) + .filter(String::isNotBlank) + .joinToString(" ") + currentText = "" + onPartial(currentTranscript()) + } + }) + } + transcriber.start() + audio = Channel(Channel.UNLIMITED) + worker = scope.launch { + for (chunk in audio!!) transcriber.addAudio(chunk, 16_000) + } + } + + override fun acceptAudio(samples: FloatArray) { + audio?.trySend(samples) + } + + override suspend fun finishStreaming(): String { + audio?.close() + worker?.join() + withContext(Dispatchers.Default) { transcriberOrThrow().stop() } + audio = null + worker = null + return currentTranscript() + } + + override suspend fun close() { + val wasStreaming = audio != null + audio?.close() + worker?.cancelAndJoin() + if (wasStreaming) { + runCatching { withContext(Dispatchers.Default) { transcriber?.stop() } } + } + audio = null + worker = null + transcriber?.removeAllListeners() + transcriber = null + scope.cancel() + } + + private fun currentTranscript() = listOf(completedText, currentText) + .filter(String::isNotBlank) + .joinToString(" ") + .trim() + + private fun transcriberOrThrow() = + transcriber ?: throw IllegalStateException("Moonshine backend is not loaded") +} diff --git a/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineModel.kt b/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineModel.kt new file mode 100644 index 0000000..3629486 --- /dev/null +++ b/app/src/main/java/org/futo/voiceinput/moonshine/MoonshineModel.kt @@ -0,0 +1,61 @@ +package org.futo.voiceinput.moonshine + +import android.content.Context +import android.content.Intent +import org.futo.voiceinput.downloader.DownloadActivity +import org.futo.voiceinput.downloader.EXTRA_COMPLETION_MARKER +import org.futo.voiceinput.downloader.EXTRA_DOWNLOAD_FILE_HASHES +import org.futo.voiceinput.downloader.EXTRA_DOWNLOAD_FILE_NAMES +import org.futo.voiceinput.downloader.EXTRA_DOWNLOAD_FILE_URLS +import org.futo.voiceinput.downloader.EXTRA_TARGET_SUBDIR +import java.io.File + +object MoonshineModel { + const val directoryName = "moonshine-small-streaming-en" + const val completionMarker = ".download_complete" + private const val baseUrl = + "https://download.moonshine.ai/model/small-streaming-en/quantized" + + val files = listOf( + "adapter.ort", + "cross_kv.ort", + "decoder_kv.ort", + "decoder_kv_with_attention.ort", + "encoder.ort", + "frontend.ort", + "streaming_config.json", + "tokenizer.bin" + ) + + fun url(file: String) = "$baseUrl/$file" +} + +fun Context.moonshineModelDir(): File = File(filesDir, MoonshineModel.directoryName) + +fun Context.isMoonshineModelDownloaded(): Boolean { + val directory = moonshineModelDir() + return File(directory, MoonshineModel.completionMarker).exists() && + MoonshineModel.files.all { File(directory, it).exists() } +} + +fun Context.moonshineModelDownloadIntent(): Intent = + Intent(this, DownloadActivity::class.java).apply { + putExtra(EXTRA_TARGET_SUBDIR, MoonshineModel.directoryName) + putExtra(EXTRA_COMPLETION_MARKER, MoonshineModel.completionMarker) + putStringArrayListExtra( + EXTRA_DOWNLOAD_FILE_NAMES, + ArrayList(MoonshineModel.files) + ) + putStringArrayListExtra( + EXTRA_DOWNLOAD_FILE_URLS, + ArrayList(MoonshineModel.files.map(MoonshineModel::url)) + ) + putStringArrayListExtra( + EXTRA_DOWNLOAD_FILE_HASHES, + ArrayList(List(MoonshineModel.files.size) { "" }) + ) + } + +fun Context.startMoonshineModelDownloadActivity() { + startActivity(moonshineModelDownloadIntent()) +} diff --git a/app/src/main/java/org/futo/voiceinput/parakeet/ParakeetBackend.kt b/app/src/main/java/org/futo/voiceinput/parakeet/ParakeetBackend.kt index 50adbd3..8d26b60 100644 --- a/app/src/main/java/org/futo/voiceinput/parakeet/ParakeetBackend.kt +++ b/app/src/main/java/org/futo/voiceinput/parakeet/ParakeetBackend.kt @@ -1,6 +1,7 @@ package org.futo.voiceinput.parakeet import android.content.Context +import org.futo.voiceinput.backend.SpeechBackend import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext diff --git a/app/src/main/java/org/futo/voiceinput/parakeet/SpeechBackend.kt b/app/src/main/java/org/futo/voiceinput/parakeet/SpeechBackend.kt deleted file mode 100644 index c5bc1a7..0000000 --- a/app/src/main/java/org/futo/voiceinput/parakeet/SpeechBackend.kt +++ /dev/null @@ -1,9 +0,0 @@ -package org.futo.voiceinput.parakeet - -import android.content.Context - -interface SpeechBackend { - suspend fun load(context: Context) - suspend fun transcribe(samples: FloatArray): String - suspend fun close() -} diff --git a/app/src/main/java/org/futo/voiceinput/settings/Settings.kt b/app/src/main/java/org/futo/voiceinput/settings/Settings.kt index 163c077..043481e 100644 --- a/app/src/main/java/org/futo/voiceinput/settings/Settings.kt +++ b/app/src/main/java/org/futo/voiceinput/settings/Settings.kt @@ -117,6 +117,7 @@ val ENABLE_30S_LIMIT = SettingsKey(booleanPreferencesKey("enable_30s_limit"), fa enum class SpeechBackendType(val id: String) { Parakeet("parakeet"), + Moonshine("moonshine"), WhisperGGML("whisper_ggml") } @@ -124,7 +125,7 @@ fun String.toSpeechBackendType(): SpeechBackendType { return SpeechBackendType.values().firstOrNull { it.id == this } ?: SpeechBackendType.Parakeet } -val SPEECH_BACKEND = SettingsKey(stringPreferencesKey("speech_backend"), SpeechBackendType.Parakeet.id) +val SPEECH_BACKEND = SettingsKey(stringPreferencesKey("speech_backend"), SpeechBackendType.Moonshine.id) val PARAKEET_KEEP_WARM = SettingsKey(booleanPreferencesKey("parakeet_keep_warm"), true) val PARAKEET_KEEP_WARM_TIMEOUT_MS = SettingsKey(longPreferencesKey("parakeet_keep_warm_timeout_ms"), 5 * 60 * 1000L) diff --git a/app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt b/app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt index 3d28275..1500940 100644 --- a/app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt +++ b/app/src/main/java/org/futo/voiceinput/settings/pages/Models.kt @@ -33,6 +33,8 @@ import org.futo.voiceinput.migration.ConditionalModelUpdate import org.futo.voiceinput.migration.NeedsMigration import org.futo.voiceinput.parakeet.isParakeetModelDownloaded import org.futo.voiceinput.parakeet.startParakeetModelDownloadActivity +import org.futo.voiceinput.moonshine.isMoonshineModelDownloaded +import org.futo.voiceinput.moonshine.startMoonshineModelDownloadActivity import org.futo.voiceinput.settings.DISMISS_MIGRATION_TIP import org.futo.voiceinput.settings.ENABLE_MULTILINGUAL import org.futo.voiceinput.settings.ENGLISH_MODEL_INDEX @@ -69,6 +71,13 @@ fun modelsSubtitle(): String? { stringResource(R.string.parakeet_model_download_required) } } + SpeechBackendType.Moonshine -> { + if (context.isMoonshineModelDownloaded()) { + stringResource(R.string.moonshine_model_active_subtitle) + } else { + stringResource(R.string.moonshine_model_download_required) + } + } SpeechBackendType.WhisperGGML -> stringResource(R.string.whisper_ggml_model_active_subtitle) } } @@ -142,6 +151,41 @@ fun ParakeetModelStatus() { Tip(stringResource(R.string.parakeet_download_model_tip)) } +@Composable +fun MoonshineModelStatus() { + val context = LocalContext.current + val lifecycleOwner = LocalLifecycleOwner.current + val isDownloaded = remember { mutableStateOf(context.isMoonshineModelDownloaded()) } + + DisposableEffect(lifecycleOwner) { + val observer = LifecycleEventObserver { _, event -> + if (event == Lifecycle.Event.ON_RESUME) { + isDownloaded.value = context.isMoonshineModelDownloaded() + } + } + lifecycleOwner.lifecycle.addObserver(observer) + onDispose { lifecycleOwner.lifecycle.removeObserver(observer) } + } + + ScreenTitle(stringResource(R.string.moonshine_model)) + SettingItem( + title = stringResource(R.string.moonshine_streaming_model_name), + subtitle = if (isDownloaded.value) { + stringResource(R.string.moonshine_model_downloaded) + } else { + stringResource(R.string.moonshine_model_download_required) + }, + onClick = { + if (!isDownloaded.value) context.startMoonshineModelDownloadActivity() + }, + icon = { + RadioButton(selected = isDownloaded.value, onClick = null, enabled = false) + }, + disabled = false + ) { } + Tip(stringResource(R.string.moonshine_download_model_tip)) +} + @Composable fun WhisperModelRadio( title: String, @@ -229,7 +273,7 @@ fun ModelsScreen( ) { val (languages, _) = useDataStore(LANGUAGE_TOGGLES) val (backend, _) = useDataStore(SPEECH_BACKEND) - val parakeetSelected = isParakeetSelected() + val whisperSelected = backend.toSpeechBackendType() == SpeechBackendType.WhisperGGML val needsUpdate = NeedsMigration() @@ -239,7 +283,7 @@ fun ModelsScreen( ScrollableList { ScreenTitle(stringResource(R.string.model_options), showBack = true, navController = navController) - if (!parakeetSelected) { + if (whisperSelected) { ConditionalModelUpdate() if(wasMigrated.value && !dismissMigrationTip.value) { @@ -254,18 +298,21 @@ fun ModelsScreen( ) } - if(!needsUpdate) { - PersonalDictionaryEditor(disabled = false) - - Spacer(modifier = Modifier.height(32.dp)) - } } + PersonalDictionaryEditor(disabled = false) + Spacer(modifier = Modifier.height(32.dp)) + SettingRadio( title = stringResource(R.string.speech_backend), - options = listOf(SpeechBackendType.Parakeet.id, SpeechBackendType.WhisperGGML.id), + options = listOf( + SpeechBackendType.Parakeet.id, + SpeechBackendType.Moonshine.id, + SpeechBackendType.WhisperGGML.id + ), optionNames = listOf( stringResource(R.string.backend_parakeet), + stringResource(R.string.backend_moonshine), stringResource(R.string.backend_whisper_ggml) ), setting = SPEECH_BACKEND @@ -275,6 +322,7 @@ fun ModelsScreen( when (backend.toSpeechBackendType()) { SpeechBackendType.Parakeet -> ParakeetModelStatus() + SpeechBackendType.Moonshine -> MoonshineModelStatus() SpeechBackendType.WhisperGGML -> WhisperModelOptions() } } diff --git a/app/src/main/res/values/strings-crash-reporting.xml b/app/src/main/res/values/strings-crash-reporting.xml index 02ccafd..4d53ea8 100644 --- a/app/src/main/res/values/strings-crash-reporting.xml +++ b/app/src/main/res/values/strings-crash-reporting.xml @@ -1,6 +1,6 @@ - FUTO Voice Input Parakeet has crashed! Please send a report to help us fix this. + FUTO Voice Input Moonshine has crashed! Please send a report to help us fix this. Crash Reporter Send Report diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 996ab36..ee820c5 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,6 +1,6 @@ - FUTO Voice Input Parakeet - FUTO Voice Input Parakeet Method + FUTO Voice Input Moonshine + FUTO Voice Input Moonshine Method voiceinput@futo.org Suppress non-speech annotations [cough], [music], etc @@ -17,10 +17,10 @@ Thanks for the icons used within the app View Other Dependencies The authors, contributors or copyright holders listed above are not affiliated with this product and do not endorse or promote this product. Reference to the authors, contributors or copyright holders is solely for attribution purposes. Mention of their names does not imply approval or endorsement. - This is just a demonstration of how Voice Input Parakeet looks - You have installed Voice Input Parakeet and enabled the Voice input method. You should now be able to use Voice Input Parakeet within supported apps and keyboards. - When you open Voice Input Parakeet, it will look something like this: - Voice Input Parakeet will look like this + This is just a demonstration of how Voice Input Moonshine looks + You have installed Voice Input Moonshine and enabled the Voice input method. You should now be able to use Voice Input Moonshine within supported apps and keyboards. + When you open Voice Input Moonshine, it will look something like this: + Voice Input Moonshine will look like this Look for the big off-center FUTO logo in the background! Stop Recording Once you\'re done talking, you can hit the microphone button to stop @@ -48,7 +48,7 @@ Using larger model, speed may be slower Using non-default models Using non-default model - FUTO Voice Input Parakeet Settings + FUTO Voice Input Moonshine Settings Help Languages Model @@ -58,7 +58,7 @@ Will not play sounds when started / cancelled Payment Credits - Thank you for using the paid version of FUTO Voice Input Parakeet! + Thank you for using the paid version of FUTO Voice Input Moonshine! Input test field Trigger voice input Try saying something @@ -81,9 +81,9 @@ In order to use Voice Input, you need to grant microphone permission. Grant Microphone Incompatible keyboard - You appear to be using Gboard. This keyboard is incompatible with FUTO Voice Input Parakeet.\n\nGboard relies on its own voice typing system and does not appear to integrate with standard Android APIs for voice input at this time. You will need to use a different keyboard if you want to use FUTO Voice Input Parakeet with your keyboard.\n\nPlease visit the Help page in settings for more information. + You appear to be using Gboard. This keyboard is incompatible with FUTO Voice Input Moonshine.\n\nGboard relies on its own voice typing system and does not appear to integrate with standard Android APIs for voice input at this time. You will need to use a different keyboard if you want to use FUTO Voice Input Moonshine with your keyboard.\n\nPlease visit the Help page in settings for more information. I understand Gboard is incompatible - You appear to be using Typewise. This keyboard does not have a voice input button at this time, so you will need to use a different keyboard if you want to use FUTO Voice Input Parakeet with your keyboard. + You appear to be using Typewise. This keyboard does not have a voice input button at this time, so you will need to use a different keyboard if you want to use FUTO Voice Input Moonshine with your keyboard. I understand Typewise is incompatible Intent result is null or empty Intent was cancelled @@ -107,14 +107,22 @@ Download Model Speech Backend Parakeet TDT English + Moonshine v2 Streaming Whisper/GGML English-only Parakeet backend using ONNX Runtime Legacy Whisper backend using GGML models Download required The selected Whisper model must be downloaded before voice input can run. This may incur data fees on mobile data. Whisper/GGML model active - You\'ve been using FUTO Voice Input Parakeet for %d days. If you find this app useful, please consider paying to support future development of FUTO software. - Thank you for trying out FUTO Voice Input Parakeet! If you find this app useful, please consider paying to support future development of FUTO software. + Moonshine Model + Moonshine v2 Small Streaming active + Moonshine v2 Small Streaming English + Download required + Downloaded and active + Low-latency English transcription that updates while you speak. The model remains offline after download. + The Moonshine streaming model must be downloaded before voice input can run. This may incur data fees on mobile data. + You\'ve been using FUTO Voice Input Moonshine for %d days. If you find this app useful, please consider paying to support future development of FUTO software. + Thank you for trying out FUTO Voice Input Moonshine! If you find this app useful, please consider paying to support future development of FUTO software. NOTE: Development has largely shifted to FUTO Keyboard and this app is not expected to receive any major updates at this time. Nevertheless, we would appreciate your support. FUTO is dedicated to making good software that doesn\'t abuse you. This app will never serve you ads or collect your personal data. Thank you for purchasing Voice Input! @@ -166,12 +174,12 @@ Advanced About Having trouble? - Voice Input Parakeet - Voice Input Parakeet + Voice Input Moonshine + Voice Input Moonshine It looks like the default voice input app is set to a different app (%s). You may wish to change this. Clear app\'s defaults Dismiss - You appear to be using Samsung Keyboard.\n\nUnfortunately, Samsung Keyboard is currently hardcoded to only allow the choice between Samsung Voice Input and Google Voice Input. You will need to use a different keyboard if you want to trigger FUTO Voice Input Parakeet from your keyboard.\n\nPlease visit the Help page in settings for more information. + You appear to be using Samsung Keyboard.\n\nUnfortunately, Samsung Keyboard is currently hardcoded to only allow the choice between Samsung Voice Input and Google Voice Input. You will need to use a different keyboard if you want to trigger FUTO Voice Input Moonshine from your keyboard.\n\nPlease visit the Help page in settings for more information. I understand Samsung Keyboard is incompatible Payment Complete Model Downloader @@ -211,9 +219,9 @@ Dynamic Dark Theme Migration Screen - Updating your models is required to continue using FUTO Voice Input Parakeet. The old inference engine has been deprecated in March 2024, and it has now been removed, while the new inference engine brings improvements. + Updating your models is required to continue using FUTO Voice Input Moonshine. The old inference engine has been deprecated in March 2024, and it has now been removed, while the new inference engine brings improvements. Better: Personal Dictionary feature\nFaster: up to ~6x faster depending on speaking length - Thank you for supporting FUTO Voice Input Parakeet! + Thank you for supporting FUTO Voice Input Moonshine! Update has finished! To use the personal dictionary feature, visit the Model settings. If you have any feedback about the update, feel free to contact us. Model Update Update requires your attention @@ -226,13 +234,13 @@ Use beam search Recommended Models have been updated for better accuracy and faster performance. You can now also add words/phrases in the personal dictionary. If you have any issues or feedback after this change please contact us. - John Doe, Jane Smith, TensorFlow, bobble, type anything you want voice input to recognize more often here! + One preferred word or phrase per line. Use heard phrase => preferred phrase for exact corrections, for example: photo => FUTO Personal Dictionary Avoids model switching delay if you have multiple languages enabled - Unpaid FUTO Voice Input Parakeet - Pay for FUTO Voice Input Parakeet + Unpaid FUTO Voice Input Moonshine + Pay for FUTO Voice Input Moonshine Already paid? - If you already paid for FUTO Keyboard or FUTO Voice Input Parakeet, tap below. + If you already paid for FUTO Keyboard or FUTO Voice Input Moonshine, tap below. Remind later This will hide the reminder for the period of days entered. Sustainable Development diff --git a/app/src/test/java/org/futo/voiceinput/PersonalVocabularyTest.kt b/app/src/test/java/org/futo/voiceinput/PersonalVocabularyTest.kt new file mode 100644 index 0000000..af76f12 --- /dev/null +++ b/app/src/test/java/org/futo/voiceinput/PersonalVocabularyTest.kt @@ -0,0 +1,40 @@ +package org.futo.voiceinput + +import org.futo.voiceinput.settings.SpeechBackendType +import org.futo.voiceinput.settings.toSpeechBackendType +import org.junit.Assert.assertEquals +import org.junit.Test + +class PersonalVocabularyTest { + @Test + fun explicitAliasesAndVocabularyTermsCorrectTranscript() { + val vocabulary = """ + photo => FUTO + Moonshine + Parakeet TDT + """.trimIndent() + + assertEquals( + "FUTO runs Moonshine and Parakeet TDT.", + PersonalVocabulary.apply("photo runs Moonshne and parakeet tdt.", vocabulary) + ) + } + + @Test + fun shortWordsAreNotFuzzilyReplaced() { + assertEquals("cut the paper", PersonalVocabulary.apply("cut the paper", "cat")) + } + + @Test + fun legacyCommaSeparatedTermsStillWork() { + assertEquals( + "FUTO uses OpenAI and FUTO", + PersonalVocabulary.apply("futo uses Open AI and futo", "FUTO, OpenAI") + ) + } + + @Test + fun moonshineBackendIdIsParsed() { + assertEquals(SpeechBackendType.Moonshine, "moonshine".toSpeechBackendType()) + } +} diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 0000000..75129f7 --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,31 @@ +# Domain Docs + +How the engineering skills should consume this repo's domain documentation when exploring the codebase. + +## Before exploring, read these + +- **`CONTEXT.md`** at the repo root. +- **`docs/adr/`** for decisions that touch the area being changed. + +If these files don't exist, **proceed silently**. Don't flag their absence or suggest creating them upfront. The `/domain-modeling` skill creates them lazily when terms or decisions are resolved. + +## File structure + +This is a single-context repository: + +```text +/ +├── CONTEXT.md +├── docs/adr/ +└── app/ +``` + +## Use the glossary's vocabulary + +When output names a domain concept, use the term defined in `CONTEXT.md`. Don't drift to synonyms the glossary explicitly avoids. + +If the concept isn't in the glossary, reconsider whether it belongs or note the gap for `/domain-modeling`. + +## Flag ADR conflicts + +If output contradicts an existing ADR, surface the conflict explicitly rather than silently overriding it. diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 0000000..7de0f2f --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,39 @@ +# Issue tracker: GitHub + +Issues and PRDs for this repo live as GitHub issues. Use the `gh` CLI for all operations. + +## Conventions + +- **Create an issue**: `gh issue create --title "..." --body "..."`. Use a heredoc for multi-line bodies. +- **Read an issue**: `gh issue view --comments`, filtering comments by `jq` and also fetching labels. +- **List issues**: `gh issue list --state open --json number,title,body,labels,comments --jq '[.[] | {number, title, body, labels: [.labels[].name], comments: [.comments[].body]}]'` with appropriate `--label` and `--state` filters. +- **Comment on an issue**: `gh issue comment --body "..."` +- **Apply / remove labels**: `gh issue edit --add-label "..."` / `--remove-label "..."` +- **Close**: `gh issue close --comment "..."` + +Infer the repo from `git remote -v` — `gh` does this automatically when run inside a clone. + +## Pull requests as a triage surface + +**PRs as a request surface: no.** + +External PRs do not enter the triage queue. GitHub shares one number space across issues and PRs, so resolve an ambiguous `#42` with `gh pr view 42` and fall back to `gh issue view 42`. + +## When a skill says "publish to the issue tracker" + +Create a GitHub issue. + +## When a skill says "fetch the relevant ticket" + +Run `gh issue view --comments`. + +## Wayfinding operations + +Used by `/wayfinder`. The **map** is a single issue with **child** issues as tickets. + +- **Map**: a single issue labelled `wayfinder:map`, holding the Notes / Decisions-so-far / Fog body. Create it with `gh issue create --label wayfinder:map`. +- **Child ticket**: an issue linked to the map as a GitHub sub-issue. Where sub-issues aren't enabled, add the child to a task list in the map body and put `Part of #` at the top of the child body. Labels: `wayfinder:` (`research`, `prototype`, `grilling`, or `task`). Once claimed, assign the ticket to the driving developer. +- **Blocking**: use GitHub's native issue dependencies. Add an edge with `gh api --method POST repos///issues//dependencies/blocked_by -F issue_id=`, where `` is the blocker's numeric database ID from `gh api repos///issues/ --jq .id`. Where dependencies aren't available, use a `Blocked by: #, #` line at the top of the child body. +- **Frontier query**: list the map's open children, then drop tickets with an open blocker or assignee; first in map order wins. +- **Claim**: `gh issue edit --add-assignee @me` — the session's first write. +- **Resolve**: comment with the answer, close the ticket, then append a context pointer to the map's Decisions-so-far. diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 0000000..0806b2f --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,13 @@ +# Triage Labels + +The skills speak in terms of five canonical triage roles. This file maps those roles to the actual label strings used in this repo's issue tracker. + +| Label in mattpocock/skills | Label in our tracker | Meaning | +| -------------------------- | -------------------- | ---------------------------------------- | +| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue | +| `needs-info` | `needs-info` | Waiting on reporter for more information | +| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for an AFK agent | +| `ready-for-human` | `ready-for-human` | Requires human implementation | +| `wontfix` | `wontfix` | Will not be actioned | + +When a skill mentions a role, use the corresponding label string from this table. diff --git a/docs/research/liquid-ai-on-device-rewrite.md b/docs/research/liquid-ai-on-device-rewrite.md new file mode 100644 index 0000000..d1f6362 --- /dev/null +++ b/docs/research/liquid-ai-on-device-rewrite.md @@ -0,0 +1,41 @@ +# Liquid AI for on-device transcript rewriting + +Research date: 2026-07-15 + +## Recommendation + +Prototype the feature, but do not automatically rewrite every transcript. Start with **LFM2.5-350M Q4_K_M via llama.cpp**, make the model an optional download, and expose explicit actions such as Grammar, Shorter, Professional, Bullets, Email, and Text message. Preserve the original transcript and provide undo. + +Do not begin with LFM2.5-230M as the sole production model: it is smaller and faster, but Liquid positions it primarily for extraction and lightweight agentic work. Do not begin with the 1.2B or 2.6B models: their extra quality is likely outweighed by download size and peak-memory pressure beside Parakeet until device benchmarks prove otherwise. + +## Candidate models + +| Model | Q4_K_M size | Fit | +|---|---:|---| +| LFM2.5-230M | 153 MB | Useful low-end experiment; Liquid reports 213 tok/s on a Galaxy S25 Ultra, but does not recommend it for creative writing. | +| LFM2.5-350M | 229 MB in the current official repository listing | Best first prototype: instruction-tuned, multilingual, 32K context, and Liquid reports under 1 GB memory plus 188 tok/s on Snapdragon Gen 4. | +| LFM2.5-1.2B-Instruct | 731 MB | Better instruction following, but much larger and likely to create RAM pressure when Parakeet remains resident. | +| LFM2-2.6B-Transcript | Not recommended | Specialized for English 30–60 minute meeting summarization and claims under 3 GB RAM; it is the wrong task and size for short IME rewrites. | + +Sources: [230M model card](https://huggingface.co/LiquidAI/LFM2.5-230M), [230M GGUF files](https://huggingface.co/LiquidAI/LFM2.5-230M-GGUF), [350M model card](https://huggingface.co/LiquidAI/LFM2.5-350M), [350M GGUF files](https://huggingface.co/LiquidAI/LFM2.5-350M-GGUF/tree/main), [1.2B model card](https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct), [1.2B GGUF files](https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct-GGUF/tree/main), and [2.6B Transcript model card](https://huggingface.co/LiquidAI/LFM2-2.6B-Transcript). + +## Runtime choice + +Liquid officially supports GGUF/llama.cpp, ONNX Runtime, and its LEAP SDK. For this project, llama.cpp is the least risky prototype path because it is open source, supports the official Q4 models, and keeps Android 8–11 compatibility possible. + +The current LEAP Android slogan example uses SDK 0.10.7 and requires Android 12/API 31, target API 36, and Kotlin 2.3. This app currently has minSdk 26 and targetSdk 35, so adopting that LEAP configuration would exclude Android 8–11 and require build upgrades. LEAP SDK licensing must also be reviewed separately from the MIT-licensed example repository. [Liquid Android example](https://docs.liquid.ai/examples/android/slogan-generator), [official LEAP examples](https://github.com/Liquid4All/LeapSDK-Examples), and [Liquid deployment FAQ](https://docs.liquid.ai/lfm/help/faqs). + +The existing ONNX Runtime dependency is not sufficient by itself: text generation also needs tokenization, sampling, stopping, and KV-cache management. ONNX Runtime GenAI supplies those pieces, but its published architecture support list does not currently name LFM2, so compatibility should not be assumed. [ONNX Runtime GenAI](https://github.com/microsoft/onnxruntime-genai). + +## Product constraints + +- Run rewriting only after the user selects an action; never silently replace dictated text. +- Keep the raw transcript and one-tap undo because a language model can change names, numbers, negation, or intent. +- Use low-temperature generation, output-only prompts, strict token limits, and instructions to preserve names, numbers, and meaning. +- Avoid holding Parakeet and the rewrite model resident together on low-memory devices. The prototype should measure sequential unload/load versus concurrent residency. +- Make the rewrite model optional and separately downloadable; the base voice-input download should not grow by hundreds of megabytes. +- Benchmark exact transcript transformations on representative arm64 devices before choosing 230M, 350M, or 1.2B. + +## License + +The weights use the **LFM Open License v1.0**, not Apache or MIT. It permits redistribution with license/notice obligations, but commercial use by a legal entity with annual revenue of at least USD 10 million is outside the grant. Shipping therefore needs a license review, particularly if changes are intended for upstream FUTO distribution. [Official LFM license](https://huggingface.co/LiquidAI/LFM2.5-350M/blob/main/LICENSE).