diff --git a/android/app/src/main/cpp/gguf_runtime.cpp b/android/app/src/main/cpp/gguf_runtime.cpp index 8c7c705..90db9d2 100644 --- a/android/app/src/main/cpp/gguf_runtime.cpp +++ b/android/app/src/main/cpp/gguf_runtime.cpp @@ -431,9 +431,10 @@ static CreateResult try_create_session(const char* path, int n_gpu_layers, int n cparams.abort_callback = abort_callback; cparams.abort_callback_data = &operation->abort_flag; if (n_gpu_layers > 0) { - cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; - cparams.type_k = GGML_TYPE_F16; - cparams.type_v = GGML_TYPE_F16; + // Standard attention hard-aborts on some Vulkan GPUs (uncatchable); flash attn avoids it and requires quantized KV. + cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + cparams.type_k = GGML_TYPE_Q4_0; + cparams.type_v = GGML_TYPE_Q4_0; cparams.offload_kqv = true; } else { cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; @@ -792,6 +793,8 @@ Java_dev_androidagent_localmodel_gguf_GgufNative_generate( sampler_guard.smpl = llama_sampler_chain_init(sparams); llama_sampler_chain_add(sampler_guard.smpl, llama_sampler_init_top_k(static_cast(top_k))); llama_sampler_chain_add(sampler_guard.smpl, llama_sampler_init_top_p(static_cast(top_p), 1)); + // Prevents small-model repetition loops; must come after top-k/top-p per llama.cpp's guidance. + llama_sampler_chain_add(sampler_guard.smpl, llama_sampler_init_penalties(64, 1.1f, 0.0f, 0.0f)); llama_sampler_chain_add(sampler_guard.smpl, llama_sampler_init_temp(static_cast(temperature))); llama_sampler_chain_add(sampler_guard.smpl, llama_sampler_init_dist(static_cast(time(nullptr)))); diff --git a/android/app/src/main/java/dev/androidagent/AppShellActivity.kt b/android/app/src/main/java/dev/androidagent/AppShellActivity.kt index 6e60e21..5447e87 100644 --- a/android/app/src/main/java/dev/androidagent/AppShellActivity.kt +++ b/android/app/src/main/java/dev/androidagent/AppShellActivity.kt @@ -33,6 +33,7 @@ import androidx.core.view.WindowInsetsControllerCompat import androidx.lifecycle.lifecycleScope import dev.androidagent.chat.ChatAttachmentKind import dev.androidagent.chat.ChatAttachmentStore +import dev.androidagent.localmodel.LocalModelImportStatus import dev.androidagent.localmodel.LocalModelStore import dev.androidagent.settings.DiagnosticsEventLog import dev.androidagent.settings.DiagnosticsEventLevel @@ -102,6 +103,7 @@ class AppShellActivity : ComponentActivity() { } localModelImportJob?.cancel() DiagnosticsEventLog.append(DiagnosticsEventLevel.Info, "Importing local model") + LocalModelImportStatus.publish("Importing local model…") localModelImportJob = lifecycleScope.launch { try { var lastProgressMarker = -1L @@ -116,14 +118,21 @@ class AppShellActivity : ComponentActivity() { "${copied / (1024 * 1024)} MB" } DiagnosticsEventLog.append(DiagnosticsEventLevel.Info, "Importing local model: $label") + LocalModelImportStatus.publish("Importing local model… $label") } } pendingLocalModelPathField?.setText(path) DiagnosticsEventLog.append(DiagnosticsEventLevel.Success, "Imported local model") + LocalModelImportStatus.publish("Imported ✓ ${LocalModelStore.displayName(path)}") + mainHandler.postDelayed({ LocalModelImportStatus.publish(null) }, 2500) } catch (cancelled: CancellationException) { + LocalModelImportStatus.publish(null) throw cancelled } catch (error: Exception) { - DiagnosticsEventLog.append(DiagnosticsEventLevel.Error, error.message ?: "Import failed") + val message = error.message ?: "Import failed" + DiagnosticsEventLog.append(DiagnosticsEventLevel.Error, message) + LocalModelImportStatus.publish("Import failed: $message") + mainHandler.postDelayed({ LocalModelImportStatus.publish(null) }, 4000) } } } diff --git a/android/app/src/main/java/dev/androidagent/MainActivity.kt b/android/app/src/main/java/dev/androidagent/MainActivity.kt index a016b94..5f26770 100644 --- a/android/app/src/main/java/dev/androidagent/MainActivity.kt +++ b/android/app/src/main/java/dev/androidagent/MainActivity.kt @@ -28,6 +28,7 @@ import androidx.core.view.WindowInsetsCompat import androidx.core.view.WindowInsetsControllerCompat import androidx.lifecycle.lifecycleScope import dev.androidagent.avatar.AvatarLibrary +import dev.androidagent.localmodel.LocalModelImportStatus import dev.androidagent.localmodel.LocalModelStore import dev.androidagent.settings.SettingsHost import dev.androidagent.settings.SettingsUi @@ -67,6 +68,7 @@ class MainActivity : ComponentActivity() { } localModelImportJob?.cancel() setupBanner.text = "Importing local model..." + LocalModelImportStatus.publish("Importing local model…") localModelImportJob = lifecycleScope.launch { try { var lastProgressMarker = -1L @@ -74,14 +76,23 @@ class MainActivity : ComponentActivity() { val marker = if (total != null && total > 0L) copied * 100L / total else copied / PROGRESS_STEP_BYTES if (marker == lastProgressMarker) return@importModel lastProgressMarker = marker - mainHandler.post { setupBanner.text = modelImportProgress(copied, total) } + mainHandler.post { + setupBanner.text = modelImportProgress(copied, total) + LocalModelImportStatus.publish(modelImportProgress(copied, total)) + } } pendingLocalModelPathField?.setText(path) setupBanner.text = "Imported local model." + LocalModelImportStatus.publish("Imported ✓ ${LocalModelStore.displayName(path)}") + mainHandler.postDelayed({ LocalModelImportStatus.publish(null) }, 2500) } catch (cancelled: CancellationException) { + LocalModelImportStatus.publish(null) throw cancelled } catch (error: Exception) { - setupBanner.text = error.message ?: "Could not import local model." + val message = error.message ?: "Could not import local model." + setupBanner.text = message + LocalModelImportStatus.publish("Import failed: $message") + mainHandler.postDelayed({ LocalModelImportStatus.publish(null) }, 4000) } } } diff --git a/android/app/src/main/java/dev/androidagent/localmodel/LocalAgentController.kt b/android/app/src/main/java/dev/androidagent/localmodel/LocalAgentController.kt index c38b12d..831382d 100644 --- a/android/app/src/main/java/dev/androidagent/localmodel/LocalAgentController.kt +++ b/android/app/src/main/java/dev/androidagent/localmodel/LocalAgentController.kt @@ -115,7 +115,8 @@ class LocalAgentController( val systemPrompt = LocalPromptBuilder.systemPrompt( basePrompt = config.systemPrompt, toolsAllowed = toolsAllowed, - toolDescriptionsJson = tools.toolDescriptions(runtimeProfile, toolAccess).toString() + toolDescriptionsJson = tools.toolDescriptions(runtimeProfile, toolAccess).toString(), + isTinyModel = LocalModelSize.isTiny(LocalModelStore.displayName(config.localModelPath)) ) val transcript = selectNewestHistory( history = history, diff --git a/android/app/src/main/java/dev/androidagent/localmodel/LocalModelImportStatus.kt b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelImportStatus.kt new file mode 100644 index 0000000..cc7549d --- /dev/null +++ b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelImportStatus.kt @@ -0,0 +1,19 @@ +package dev.androidagent.localmodel + +// Only one screen is visible at a time, so a single overwrite-on-build slot is enough. +object LocalModelImportStatus { + @Volatile + private var listener: ((String?) -> Unit)? = null + + fun observe(onUpdate: (String?) -> Unit) { + listener = onUpdate + } + + fun clearObserver(onUpdate: (String?) -> Unit) { + if (listener === onUpdate) listener = null + } + + fun publish(status: String?) { + listener?.invoke(status) + } +} diff --git a/android/app/src/main/java/dev/androidagent/localmodel/LocalModelSize.kt b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelSize.kt new file mode 100644 index 0000000..2a9b81e --- /dev/null +++ b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelSize.kt @@ -0,0 +1,14 @@ +package dev.androidagent.localmodel + +// Parses param count from the display name; file size isn't reliable across quant schemes. +internal object LocalModelSize { + private val PARAM_COUNT = Regex("""(\d+(?:\.\d+)?)\s*[Bb](?:\b|-)""") + + // Below this, the tool-call JSON protocol degenerates into non-answers. + private const val TINY_THRESHOLD_BILLIONS = 2.0 + + fun isTiny(displayName: String): Boolean { + val billions = PARAM_COUNT.find(displayName)?.groupValues?.get(1)?.toDoubleOrNull() ?: return false + return billions < TINY_THRESHOLD_BILLIONS + } +} diff --git a/android/app/src/main/java/dev/androidagent/localmodel/LocalModelStore.kt b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelStore.kt index 44f44c5..485b0fa 100644 --- a/android/app/src/main/java/dev/androidagent/localmodel/LocalModelStore.kt +++ b/android/app/src/main/java/dev/androidagent/localmodel/LocalModelStore.kt @@ -10,6 +10,7 @@ import dev.androidagent.storage.StoredBlob import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.isActive import kotlinx.coroutines.withContext +import org.json.JSONObject import java.io.File import java.io.IOException import java.io.InputStream @@ -25,7 +26,8 @@ object LocalModelStore { ) private val GGUF_LIMITS = modelLimits( maxItemBytes = 12L * 1024L * 1024L * 1024L, - maxAggregateBytes = 24L * 1024L * 1024L * 1024L + maxAggregateBytes = 24L * 1024L * 1024L * 1024L, + maxBlobCount = 8 ) suspend fun importModel( @@ -102,10 +104,53 @@ object LocalModelStore { formatForDisplayName(path) != null && File(path.trim()).isFile fun displayName(path: String): String { - val name = File(path.trim()).name - return name.removeSuffix(".litertlm").removeSuffix(".gguf").ifBlank { "Local model" } + val trimmed = path.trim() + if (trimmed.isBlank()) return "No model selected" + val file = File(trimmed) + val fallback = file.name.removeSuffix(".litertlm").removeSuffix(".gguf").ifBlank { "Local model" } + val metadataFile = File(file.parentFile, "${file.nameWithoutExtension}.json") + val storedDisplayName = runCatching { + if (metadataFile.isFile) { + JSONObject(metadataFile.readText()).optString("displayName").takeIf { it.isNotBlank() } + } else { + null + } + }.getOrNull() + return storedDisplayName ?: fallback } + // Newest first. + internal fun listImportedModels(context: Context): List { + return ModelFormat.values().flatMap { format -> + val directory = File(context.filesDir, format.directoryName) + directory.listFiles().orEmpty() + .filter { it.name.endsWith(".json") } + .mapNotNull { metadataFile -> + runCatching { + val json = JSONObject(metadataFile.readText()) + val id = json.getString("id") + val payloadFile = File(directory, "$id${format.extension}") + if (!payloadFile.isFile) return@runCatching null + ImportedModel( + path = payloadFile.absolutePath, + displayName = json.optString("displayName").takeIf { it.isNotBlank() } ?: id, + format = format, + sizeBytes = json.optLong("sizeBytes", payloadFile.length()), + createdAt = json.optLong("createdAt", payloadFile.lastModified()) + ) + }.getOrNull() + } + }.sortedByDescending { it.createdAt } + } + + internal data class ImportedModel( + val path: String, + val displayName: String, + val format: ModelFormat, + val sizeBytes: Long, + val createdAt: Long + ) + internal enum class ModelFormat( val extension: String, val directoryName: String, @@ -121,10 +166,10 @@ object LocalModelStore { val mimeType: String? ) - private fun modelLimits(maxItemBytes: Long, maxAggregateBytes: Long) = BlobStoreLimits( + private fun modelLimits(maxItemBytes: Long, maxAggregateBytes: Long, maxBlobCount: Int = 3) = BlobStoreLimits( minItemBytes = MIN_MODEL_BYTES, maxItemBytes = maxItemBytes, - maxBlobCount = 3, + maxBlobCount = maxBlobCount, maxAggregateBytes = maxAggregateBytes, freeSpaceReserveBytes = 512L * 1024L * 1024L, retentionMillis = Long.MAX_VALUE diff --git a/android/app/src/main/java/dev/androidagent/localmodel/LocalPromptBuilder.kt b/android/app/src/main/java/dev/androidagent/localmodel/LocalPromptBuilder.kt index 42a72a8..8ada9ce 100644 --- a/android/app/src/main/java/dev/androidagent/localmodel/LocalPromptBuilder.kt +++ b/android/app/src/main/java/dev/androidagent/localmodel/LocalPromptBuilder.kt @@ -4,8 +4,13 @@ internal object LocalPromptBuilder { fun systemPrompt( basePrompt: String, toolsAllowed: Boolean, - toolDescriptionsJson: String + toolDescriptionsJson: String, + isTinyModel: Boolean = false ): String { + // Sub-2B models can't reliably use the tool-call protocol below; skip to a plain prompt. + if (isTinyModel) { + return "Answer the user's question directly and completely, in plain conversational language." + } val toolPolicy = if (toolsAllowed) { """ Tool mode: diff --git a/android/app/src/main/java/dev/androidagent/settings/screens/LocalModelSettingsScreen.kt b/android/app/src/main/java/dev/androidagent/settings/screens/LocalModelSettingsScreen.kt index 1ef9768..31fd15c 100644 --- a/android/app/src/main/java/dev/androidagent/settings/screens/LocalModelSettingsScreen.kt +++ b/android/app/src/main/java/dev/androidagent/settings/screens/LocalModelSettingsScreen.kt @@ -9,6 +9,8 @@ import android.widget.LinearLayout import dev.androidagent.AgentConfigStore import dev.androidagent.LocalModelBackend import dev.androidagent.R +import dev.androidagent.localmodel.LocalModelImportStatus +import dev.androidagent.localmodel.LocalModelStore import dev.androidagent.settings.ColorUtils import dev.androidagent.settings.SettingsComponents import dev.androidagent.settings.SettingsComponents.BadgeTone @@ -75,8 +77,46 @@ object LocalModelSettingsScreen { exposeToAccessibility(R.id.openclaw_local_model_path_field, "Local model path") } + val modelPickerPaths = mutableListOf() + fun modelPickerChoices(): List { + val models = LocalModelStore.listImportedModels(activity) + modelPickerPaths.clear() + modelPickerPaths.addAll(models.map { it.path }) + return if (models.isEmpty()) { + listOf("No models imported yet") + } else { + models.map { "${it.displayName} (${it.sizeBytes / (1024 * 1024)} MB)" } + } + } + val initialModelChoices = modelPickerChoices() + val initialModelSelection = modelPickerPaths.indexOf(config.localModelPath).coerceAtLeast(0) + val modelPickerSpinner = SettingsUi.styledSpinner(activity, initialModelChoices, initialModelSelection, tokens) + root.addView(modelPickerSpinner, SettingsComponents.verticalMargin(activity, bottom = DesignTokens.Spacing.md)) + SettingsUi.onSpinnerSelectionChanged(modelPickerSpinner) { index -> + modelPickerPaths.getOrNull(index)?.let { path -> pathInput.setText(path) } + } + // Import model card - root.addView(buildImportCard(activity, tokens, callbacks, pathInput, config.localModelPath)) + val importProgressLabel = SettingsComponents.body(activity, tokens, "").apply { + visibility = View.GONE + } + root.addView(buildImportCard(activity, tokens, callbacks, pathInput, config.localModelPath, importProgressLabel)) + LocalModelImportStatus.observe { status -> + if (status == null) { + importProgressLabel.visibility = View.GONE + val choices = modelPickerChoices() + (modelPickerSpinner.adapter as? android.widget.ArrayAdapter)?.let { adapter -> + adapter.clear() + adapter.addAll(choices) + adapter.notifyDataSetChanged() + } + val selection = modelPickerPaths.indexOf(pathInput.text.toString()).coerceAtLeast(0) + modelPickerSpinner.setSelection(selection) + } else { + importProgressLabel.visibility = View.VISIBLE + importProgressLabel.text = status + } + } // Backend card val backends = LocalModelBackend.values().toList() @@ -158,7 +198,8 @@ object LocalModelSettingsScreen { tokens: ThemeTokens, callbacks: Callbacks, pathInput: EditText, - currentPath: String + currentPath: String, + progressLabel: android.widget.TextView ): LinearLayout { val card = SettingsComponents.card(activity, tokens, padding = DesignTokens.Spacing.md) val row = LinearLayout(activity).apply { @@ -191,6 +232,7 @@ object LocalModelSettingsScreen { }.exposeToAccessibility(R.id.openclaw_local_model_import_button, "Import local model")) card.addView(row) + card.addView(progressLabel, SettingsComponents.verticalMargin(activity, top = DesignTokens.Spacing.sm)) // Hidden text field driver card.addView(pathInput) return card diff --git a/android/app/src/main/java/dev/androidagent/settings/screens/RuntimeSettingsScreen.kt b/android/app/src/main/java/dev/androidagent/settings/screens/RuntimeSettingsScreen.kt index 950e946..10e2d37 100644 --- a/android/app/src/main/java/dev/androidagent/settings/screens/RuntimeSettingsScreen.kt +++ b/android/app/src/main/java/dev/androidagent/settings/screens/RuntimeSettingsScreen.kt @@ -17,6 +17,8 @@ import dev.androidagent.LocalModelBackend import dev.androidagent.R import dev.androidagent.chat.ChatModelCatalog import dev.androidagent.chat.ChatModelOption +import dev.androidagent.localmodel.LocalModelImportStatus +import dev.androidagent.localmodel.LocalModelStore import dev.androidagent.settings.DiagnosticsBackendSnapshot import dev.androidagent.settings.SettingsButtonTone import dev.androidagent.settings.SettingsUi @@ -81,7 +83,23 @@ object RuntimeSettingsScreen { val workspaceControls = root.addWorkspaceCards(activity, tokens, config) val localModelPathInput = SettingsUi.configField(activity, "Model file", config.localModelPath, tokens).apply { - exposeToAccessibility(R.id.openclaw_local_model_path_field, "Local model path") + visibility = View.GONE + } + val modelPickerPaths = mutableListOf() + fun modelPickerChoices(): List { + val models = LocalModelStore.listImportedModels(activity) + modelPickerPaths.clear() + modelPickerPaths.addAll(models.map { it.path }) + return if (models.isEmpty()) { + listOf("No models imported yet") + } else { + models.map { "${it.displayName} (${it.sizeBytes / (1024 * 1024)} MB)" } + } + } + val initialModelChoices = modelPickerChoices() + val initialModelSelection = modelPickerPaths.indexOf(config.localModelPath).coerceAtLeast(0) + val modelPickerSpinner = SettingsUi.styledSpinner(activity, initialModelChoices, initialModelSelection, tokens).apply { + exposeToAccessibility(R.id.openclaw_local_model_path_field, "Imported model") } val localBackends = LocalModelBackend.values().toList() val localBackendSpinner = SettingsUi.styledSpinner( @@ -108,15 +126,37 @@ object RuntimeSettingsScreen { R.id.openclaw_local_developer_tools_checkbox ) + val importProgressLabel = SettingsUi.body(activity, "", tokens).apply { + visibility = View.GONE + } + LocalModelImportStatus.observe { status -> + if (status == null) { + importProgressLabel.visibility = View.GONE + val choices = modelPickerChoices() + (modelPickerSpinner.adapter as? android.widget.ArrayAdapter)?.let { adapter -> + adapter.clear() + adapter.addAll(choices) + adapter.notifyDataSetChanged() + } + val selection = modelPickerPaths.indexOf(localModelPathInput.text.toString()).coerceAtLeast(0) + modelPickerSpinner.setSelection(selection) + } else { + importProgressLabel.visibility = View.VISIBLE + importProgressLabel.text = status + } + } + root.addView(SettingsUi.card(activity, tokens).apply { addView(SettingsUi.sectionHeader(activity, "Local Models", "Import and tune the on-device model harness.", tokens)) - addView(SettingsUi.labeledField(activity, "Model file", localModelPathInput, tokens, DesignTokens.Spacing.md)) + addView(SettingsUi.labeledField(activity, "Model", modelPickerSpinner, tokens, DesignTokens.Spacing.md)) addView( SettingsUi.actionButton(activity, "Import Local Model", SettingsButtonTone.Secondary, tokens) { callbacks.onImportRequested(localModelPathInput) }.exposeToAccessibility(R.id.openclaw_local_model_import_button, "Import local model"), SettingsUi.stackedParams(activity, DesignTokens.Spacing.sm + 2) ) + addView(importProgressLabel, SettingsUi.stackedParams(activity, DesignTokens.Spacing.sm)) + addView(localModelPathInput) addView(SettingsUi.labeledField(activity, "Backend", localBackendSpinner, tokens)) addView(SettingsUi.labeledField(activity, "Context window", localContextInput, tokens)) addView(localDeveloperTools, SettingsUi.stackedParams(activity, DesignTokens.Spacing.md)) @@ -181,6 +221,9 @@ object RuntimeSettingsScreen { SettingsUi.onTextChanged(control.input) { saveCurrent() } } SettingsUi.onTextChanged(localModelPathInput) { saveCurrent() } + SettingsUi.onSpinnerSelectionChanged(modelPickerSpinner) { index -> + modelPickerPaths.getOrNull(index)?.let { path -> localModelPathInput.setText(path) } + } SettingsUi.onSpinnerSelectionChanged(localBackendSpinner) { saveCurrent() } SettingsUi.onTextChanged(localContextInput) { saveCurrent() } diff --git a/pc/package-lock.json b/pc/package-lock.json index 0bc6260..3a8b6fe 100644 --- a/pc/package-lock.json +++ b/pc/package-lock.json @@ -546,7 +546,7 @@ "typebox": "1.1.38" }, "bin": { - "pi-ai": "./dist/cli.js" + "pi-ai": "dist/cli.js" }, "engines": { "node": ">=22.19.0"