From 4905c775bd8bc3081f91a6ee85c83c8a840887e9 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 11:56:58 +0800 Subject: [PATCH 01/14] feat(llm): add per-request model usage records for Rust and Python consumers --- .github/workflows/ci.yml | 15 + AGENTS.md | 4 +- Cargo.lock | 10 + Cargo.toml | 3 + pylib/robonix-api/pyproject.toml | 2 +- pylib/robonix-api/robonix_api/llm/README.md | 72 ++++- pylib/robonix-api/robonix_api/llm/__init__.py | 5 +- .../robonix_api/llm/contract/usage_cases.json | 94 ++++++ pylib/robonix-api/robonix_api/llm/metrics.py | 171 ++++++++++ pylib/robonix-api/tests/test_llm_metrics.py | 136 ++++++++ system/llm-metrics/Cargo.toml | 15 + system/llm-metrics/README.md | 25 ++ system/llm-metrics/src/lib.rs | 292 ++++++++++++++++++ system/llm-metrics/tests/contract.rs | 68 ++++ 14 files changed, 905 insertions(+), 7 deletions(-) create mode 100644 pylib/robonix-api/robonix_api/llm/contract/usage_cases.json create mode 100644 pylib/robonix-api/robonix_api/llm/metrics.py create mode 100644 pylib/robonix-api/tests/test_llm_metrics.py create mode 100644 system/llm-metrics/Cargo.toml create mode 100644 system/llm-metrics/README.md create mode 100644 system/llm-metrics/src/lib.rs create mode 100644 system/llm-metrics/tests/contract.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f522621cf..180cb7fce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,21 @@ jobs: system/scene/tests/test_goal_planner.py system/scene/tests/test_malformed_grid_inputs.py + robonix-api-tests: + name: robonix-api unit tests + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + # Installs the package the way a consumer gets it, so an import the + # tests rely on cannot pass by accident through PYTHONPATH alone. + - name: Install robonix-api + run: python3 -m pip install ./pylib/robonix-api + - name: Test robonix-api + run: python3 -m unittest discover -s pylib/robonix-api/tests -p 'test_*.py' -v + chronos-python: name: Chronos Python bindings runs-on: ubuntu-22.04 diff --git a/AGENTS.md b/AGENTS.md index abb17c0d7..e09e8988a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,8 +41,8 @@ Third-party skills are not vendored into the repository. These are recommended f Robonix is an **embodied AI operating system**: a monorepo with Rust system components, Python packages, capability contracts, IDL libraries, examples, and documentation. The Rust workspace lives at the repository root in `Cargo.toml`; -current Rust members are `system/{atlas,executor,liaison,pilot,scribe,soma,vitals}` -and `tools/{rbnx,codegen}`. The twelve system component directories live under +current Rust members are `system/{atlas,executor,liaison,llm-metrics,pilot,scribe,soma,vitals}` +and `tools/{rbnx,codegen}`. The thirteen system component directories live under `system/`; several are documentation/planning components today, while `system/scene` is a Python package managed by uv. Python workspace packages are listed in the root `pyproject.toml`: shared libraries in `pylib/*`, reference diff --git a/Cargo.lock b/Cargo.lock index 080f9c802..749290213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3312,6 +3312,16 @@ dependencies = [ "serde_yaml", ] +[[package]] +name = "robonix-llm-metrics" +version = "1.1.0" +dependencies = [ + "robonix-scribe", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "robonix-pilot" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 04d3db88a..75063a284 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ members = [ "system/liaison", # Shared model catalog for LLM consumers (Pilot and future users). "system/llm-catalog", + # Per-request model usage records shared by LLM consumers. + "system/llm-metrics", "system/pilot", "system/scribe", "system/soma", @@ -86,3 +88,4 @@ crossterm = "0.29.0" robonix-chronos-clock = { path = "system/chronos/clock" } robonix-codegen = { path = "tools/codegen" } robonix-scribe = { path = "system/scribe" } +robonix-llm-metrics = { path = "system/llm-metrics" } diff --git a/pylib/robonix-api/pyproject.toml b/pylib/robonix-api/pyproject.toml index 27c64dcb0..f79ac63c0 100644 --- a/pylib/robonix-api/pyproject.toml +++ b/pylib/robonix-api/pyproject.toml @@ -56,4 +56,4 @@ include = ["robonix_api*"] [tool.setuptools.package-data] robonix_api = ["_generated/*.py"] -"robonix_api.llm" = ["runtime_env.txt"] +"robonix_api.llm" = ["runtime_env.txt", "contract/*.json"] diff --git a/pylib/robonix-api/robonix_api/llm/README.md b/pylib/robonix-api/robonix_api/llm/README.md index 48a4bbe99..add2fa931 100644 --- a/pylib/robonix-api/robonix_api/llm/README.md +++ b/pylib/robonix-api/robonix_api/llm/README.md @@ -135,13 +135,82 @@ The requirements used by the in-tree consumers: The Rust implementation lives in `system/llm-catalog` and reads the same catalog with the same rules. The shared cases in `robonix_api/llm/contract/resolve_cases.json` are run by both test suites. +## Model usage records + +Shared helpers for Robonix components that call a model. `metrics` records +one JSON line per model call so usage can be counted, and later billed, +across every component. + +### Usage + +```python +from robonix_api.llm import metrics + +with metrics.Call("scene", "relations", base_url, model) as call: + response = await client.post(f"{base_url}/chat/completions", json=body) + call.finish(response) +``` + +`finish(response)` reads `status_code` and the token usage in `response.json()`; +call it for any answer, including an error status, so the attempt is counted. +`retry()` counts one more request sent for the same call. Leaving the block +without `finish` records `timeout`, `cancelled` or `transport_error` from +the exception, and the exception still propagates; so call `finish` before +`raise_for_status()`. A write failure is logged +once and never fails the model call. The first call logs whether records are +on, so an unset variable is visible in the component's log. + +### Record + +Records go to `$ROBONIX_LLM_METRICS_DIR/-.jsonl`; `consumer` +must match `[a-z0-9_-]+`, because it names the file, and `Call` raises +`ValueError` otherwise. Nothing is written when the variable is unset. +`rbnx boot` sets it to `/rbnx-boot/llm-metrics/`, which boot and +`rbnx clean -f` keep unless `--llm-metrics` is given. Each record is appended +with one write, so a reader sees whole lines except possibly the one being +written; `rbnx llm-usage` skips such a line and sums the rest. The files grow by +one line per call and are never rotated: archive or remove them with +`rbnx clean -f --llm-metrics` once their totals are taken. + +| Field | Meaning | +|---|---| +| `v` | Layout version, currently 1 | +| `ts_ms`, `latency_ms` | Wall-clock end time and request duration | +| `boot_id`, `pid` | `RBNX_BOOT_ID` of the deployment and the writing process | +| `consumer`, `operation` | Component and what it used the model for | +| `host` | Endpoint host and port; never the path, query or credentials | +| `model`, `provider_model` | Model requested, and the one the response names | +| `stream` | Whether the reply was streamed | +| `status`, `http_status` | `ok` (2xx), `http_error`, `timeout`, `cancelled` or `transport_error` (also a streamed reply cut off before it finished) | +| `attempts` | Requests sent for this call, counting retries | +| `input_tokens`, `output_tokens`, `cached_input_tokens`, `reasoning_tokens` | Reported counts; `null` when not reported | + +Prompts, replies and keys are never recorded. Prices are not recorded either: +cost is token counts times the operator's price list, joined on `host` + +`model` against the totals `rbnx llm-usage --json` prints. A launcher that +filters the environment must pass `ROBONIX_LLM_METRICS_DIR` and `RBNX_BOOT_ID` +through; otherwise its consumers write nothing, which their startup log says. + +The Rust crate `system/llm-metrics` writes the same record. Both parse usage +with the cases in [contract/usage_cases.json](contract/usage_cases.json). + +| Consumer | Operations | +|---|---| +| `pilot` | `plan`, `summary` | +| `scene` | `relations`, `image_relations`, `caption`, `perception` | +| `memory` | `search`, `observe`, `answer` | +| `memsearch` | `compact` | +| `vlm_verifier` | `judge` | + ## Tests ```bash PYTHONPATH=pylib/robonix-api python3 -m unittest discover \ -s pylib/robonix-api/tests -p 'test_llm_*.py' -v +PYTHONPATH=pylib/robonix-api python3 -m unittest pylib/robonix-api/tests/test_llm_metrics.py cargo test -p robonix-llm-catalog +cargo test -p robonix-llm-metrics ``` ## Direct-connect smoke test @@ -151,5 +220,4 @@ calls it directly with a generated image: ```bash export DEEPSEEK_API_KEY=... -PYTHONPATH=pylib/robonix-api python3 pylib/robonix-api/examples/llm_deepseek_smoke.py -``` +PYTHONPATH=pylib/robonix-api python3 pylib/robonix-api/examples/llm_deepseek_smoke.py``` diff --git a/pylib/robonix-api/robonix_api/llm/__init__.py b/pylib/robonix-api/robonix_api/llm/__init__.py index 5547af2c2..f275155f4 100644 --- a/pylib/robonix-api/robonix_api/llm/__init__.py +++ b/pylib/robonix-api/robonix_api/llm/__init__.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: MulanPSL-2.0 -"""Feature catalog and model resolver for Robonix consumers. +"""Feature catalog, model resolver and usage records for Robonix consumers. -This package answers one question: given feature requirements, which model +The catalog answers one question: given feature requirements, which model endpoint should this consumer connect to? It does not proxy traffic, track health or manage keys; the caller resolves once and connects directly. +`metrics` records one line per model request; see README.md. """ from .catalog import ModelCatalog, ModelSpec from .resolve import ( diff --git a/pylib/robonix-api/robonix_api/llm/contract/usage_cases.json b/pylib/robonix-api/robonix_api/llm/contract/usage_cases.json new file mode 100644 index 000000000..8ddc1d989 --- /dev/null +++ b/pylib/robonix-api/robonix_api/llm/contract/usage_cases.json @@ -0,0 +1,94 @@ +{ + "cases": [ + { + "name": "openai body with cache and reasoning", + "body": { + "model": "gpt-5.5-2026-01-01", + "usage": { + "prompt_tokens": 1200, + "completion_tokens": 80, + "total_tokens": 1280, + "prompt_tokens_details": { + "cached_tokens": 900 + }, + "completion_tokens_details": { + "reasoning_tokens": 32 + } + } + }, + "expected": { + "input_tokens": 1200, + "output_tokens": 80, + "cached_input_tokens": 900, + "reasoning_tokens": 32, + "provider_model": "gpt-5.5-2026-01-01" + } + }, + { + "name": "deepseek cache hit field", + "body": { + "usage": { + "prompt_tokens": 500, + "completion_tokens": 10, + "prompt_cache_hit_tokens": 400, + "prompt_cache_miss_tokens": 100 + } + }, + "expected": { + "input_tokens": 500, + "output_tokens": 10, + "cached_input_tokens": 400, + "reasoning_tokens": null, + "provider_model": null + } + }, + { + "name": "stream chunk without usage", + "body": { + "model": "m", + "choices": [ + { + "delta": { + "content": "hi" + } + } + ] + }, + "expected": { + "input_tokens": null, + "output_tokens": null, + "cached_input_tokens": null, + "reasoning_tokens": null, + "provider_model": "m" + } + }, + { + "name": "invalid counts are unknown", + "body": { + "usage": { + "prompt_tokens": -1, + "completion_tokens": "7", + "prompt_tokens_details": null + } + }, + "expected": { + "input_tokens": null, + "output_tokens": null, + "cached_input_tokens": null, + "reasoning_tokens": null, + "provider_model": null + } + }, + { + "name": "not an object", + "body": null, + "expected": { + "input_tokens": null, + "output_tokens": null, + "cached_input_tokens": null, + "reasoning_tokens": null, + "provider_model": null + } + } + ] +} diff --git a/pylib/robonix-api/robonix_api/llm/metrics.py b/pylib/robonix-api/robonix_api/llm/metrics.py new file mode 100644 index 000000000..d89d9aab6 --- /dev/null +++ b/pylib/robonix-api/robonix_api/llm/metrics.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: MulanPSL-2.0 +"""One JSON line per model request, for usage metrics and later billing. + +Records go to ``$ROBONIX_LLM_METRICS_DIR/-.jsonl``; nothing is +written when the variable is unset. Prompts, replies and keys are never +recorded, and the endpoint is reduced to its host. The Rust crate +``robonix-llm-metrics`` writes the same record; both parse usage with the +cases in ``contract/usage_cases.json``. +""" +from __future__ import annotations + +import json +import logging +import os +import re +import threading +import time +from pathlib import Path +from typing import Any, Optional + +log = logging.getLogger(__name__) + +SCHEMA_VERSION = 1 +DIRECTORY_ENV = "ROBONIX_LLM_METRICS_DIR" +_SLUG = re.compile(r"[a-z0-9_-]+") +_lock = threading.Lock() +_warned = False +_announced = False + + +def usage_from_response(body: Any) -> dict[str, Any]: + """Token counts from a Chat Completions body or stream chunk; absent ones are None.""" + usage = body.get("usage") if isinstance(body, dict) else None + if not isinstance(usage, dict): + usage = {} + prompt = usage.get("prompt_tokens_details") + completion = usage.get("completion_tokens_details") + cached = prompt.get("cached_tokens") if isinstance(prompt, dict) else None + if cached is None: + # DeepSeek reports cache hits outside prompt_tokens_details. + cached = usage.get("prompt_cache_hit_tokens") + model = body.get("model") if isinstance(body, dict) else None + return { + "input_tokens": _count(usage.get("prompt_tokens")), + "output_tokens": _count(usage.get("completion_tokens")), + "cached_input_tokens": _count(cached), + "reasoning_tokens": _count( + completion.get("reasoning_tokens") if isinstance(completion, dict) else None + ), + "provider_model": model if isinstance(model, str) and model else None, + } + + +def _count(value: Any) -> Optional[int]: + return value if type(value) is int and value >= 0 else None + + +def _host(base_url: str) -> str: + """Authority without scheme, credentials or path; the same string the Rust writer keeps.""" + rest = base_url.split("://", 1)[1] if "://" in base_url else (base_url or "") + authority = re.split(r"[/?#]", rest, maxsplit=1)[0] + return authority.rsplit("@", 1)[-1].lower() + + +class Call: + """One model request; use as a context manager around the HTTP exchange. + + ``finish`` stores the outcome. Leaving the block without it records the + exception kind (``timeout``, ``cancelled`` or ``transport_error``), so a + failed request is still counted: providers can bill those too. + """ + + def __init__( + self, + consumer: str, + operation: str, + base_url: str, + model: str, + *, + stream: bool = False, + ) -> None: + if not _SLUG.fullmatch(consumer): + raise ValueError(f"llm metrics consumer {consumer!r} must match [a-z0-9_-]+") + self.record: dict[str, Any] = { + "v": SCHEMA_VERSION, + "consumer": consumer, + "operation": operation, + "host": _host(base_url), + "model": model, + "stream": stream, + "attempts": 1, + } + self._started = 0.0 + self._done = False + + def __enter__(self) -> "Call": + self._started = time.monotonic() + return self + + def retry(self) -> None: + """Count one more request sent for this call, such as a retry.""" + self.record["attempts"] += 1 + + def finish(self, response: Any = None) -> None: + """Record an HTTP response (``status_code`` and ``json()``, as httpx has). + + Without a response the call is ``ok`` with unknown usage. A body that is + not JSON leaves usage unknown; the request was still answered. + """ + status = getattr(response, "status_code", None) + try: + body = response.json() if response is not None else None + except ValueError: + body = None + self.record["status"] = ( + "ok" if status is None or 200 <= status < 300 else "http_error" + ) + self.record["http_status"] = status + self.record.update(usage_from_response(body)) + self._done = True + + def __exit__(self, exc_type, _exc, _tb) -> bool: + if not self._done: + name = exc_type.__name__ if exc_type else "" + if "Timeout" in name: + status = "timeout" + elif name == "CancelledError": + status = "cancelled" + else: + status = "transport_error" + self.record.update(status=status, http_status=None, **usage_from_response(None)) + self.record["latency_ms"] = round((time.monotonic() - self._started) * 1000) + write(self.record) + return False + + +def write(record: dict[str, Any]) -> None: + """Append ``record`` with timestamp, boot and process ids; never raises.""" + global _warned, _announced # noqa: PLW0603 + directory = os.environ.get(DIRECTORY_ENV, "").strip() + if not _announced: + # Without this line an unset variable loses every record silently. + _announced = True + log.info("llm metrics: %s", f"records -> {directory}" if directory + else f"off ({DIRECTORY_ENV} unset)") + if not directory: + return + line = dict(record, ts_ms=int(time.time() * 1000), + boot_id=os.environ.get("RBNX_BOOT_ID") or None, pid=os.getpid()) + path = Path(directory) / f"{record['consumer']}-{os.getpid()}.jsonl" + # "replace": a provider may name a model with a lone surrogate, which + # UTF-8 cannot encode; dropping the record would lose the usage it reports. + data = (json.dumps(line, ensure_ascii=False, separators=(",", ":")) + "\n").encode( + "utf-8", "replace") + try: + with _lock: + path.parent.mkdir(parents=True, exist_ok=True) + # One write of the whole line, so a reader or a crash never sees + # half a record. + fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o644) + try: + os.write(fd, data) + finally: + os.close(fd) + except (OSError, ValueError) as error: + # A metrics failure must not fail the model call; say so once. + with _lock: + first = not _warned + _warned = True + if first: + log.warning("llm metrics: cannot write %s: %s", path, error) diff --git a/pylib/robonix-api/tests/test_llm_metrics.py b/pylib/robonix-api/tests/test_llm_metrics.py new file mode 100644 index 000000000..6a9545910 --- /dev/null +++ b/pylib/robonix-api/tests/test_llm_metrics.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: MulanPSL-2.0 +from __future__ import annotations + +import asyncio +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from robonix_api.llm import metrics + +CASES = Path(metrics.__file__).parent / "contract" / "usage_cases.json" + + +class UsageFromResponseTest(unittest.TestCase): + def test_shared_cases(self) -> None: + for case in json.loads(CASES.read_text())["cases"]: + with self.subTest(case["name"]): + self.assertEqual(metrics.usage_from_response(case["body"]), case["expected"]) + + +class Response: + def __init__(self, status_code: int, body: object) -> None: + self.status_code = status_code + self.body = body + + def json(self) -> object: + if isinstance(self.body, str): + raise ValueError("not JSON") + return self.body + + +class CallTest(unittest.TestCase): + def setUp(self) -> None: + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + patcher = mock.patch.dict(os.environ, { + "ROBONIX_LLM_METRICS_DIR": self.directory.name, "RBNX_BOOT_ID": "boot-1", + }) + patcher.start() + self.addCleanup(patcher.stop) + + def records(self) -> list[dict]: + path = Path(self.directory.name) / f"scene-{os.getpid()}.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] + + def test_reply_is_recorded_without_credentials_or_path(self) -> None: + body = {"usage": {"prompt_tokens": 10, "completion_tokens": 2}} + with metrics.Call("scene", "relations", "https://user:k@api.example.com/v1?key=s", "m") as call: + call.finish(Response(200, body)) + [record] = self.records() + self.assertEqual(record["host"], "api.example.com") + self.assertEqual((record["status"], record["input_tokens"], record["output_tokens"]), ("ok", 10, 2)) + self.assertEqual((record["consumer"], record["operation"], record["boot_id"]), ("scene", "relations", "boot-1")) + self.assertNotIn("key", json.dumps(record)) + + def test_failures_are_recorded_and_still_raise(self) -> None: + class ReadTimeout(Exception): + pass + + with metrics.Call("scene", "a", "http://127.0.0.1:8000/v1", "m") as call: + call.finish(Response(429, {"error": {}})) + with metrics.Call("scene", "d", "", "m") as call: + call.finish(Response(200, "data: not json")) + with self.assertRaises(ReadTimeout), metrics.Call("scene", "b", "", "m"): + raise ReadTimeout() + + async def cancelled() -> None: + with metrics.Call("scene", "c", "", "m"): + raise asyncio.CancelledError() + + with self.assertRaises(asyncio.CancelledError): + asyncio.run(cancelled()) + + statuses = [(r["host"], r["status"], r["http_status"]) for r in self.records()] + self.assertEqual(statuses, [ + ("127.0.0.1:8000", "http_error", 429), ("", "ok", 200), + ("", "timeout", None), ("", "cancelled", None), + ]) + + def test_retries_are_counted(self) -> None: + with metrics.Call("scene", "a", "", "m") as call: + call.retry() + call.finish(Response(200, {})) + self.assertEqual(self.records()[0]["attempts"], 2) + + def test_consumer_must_be_a_file_name_slug(self) -> None: + for consumer in ("", "../scene", "Scene"): + with self.subTest(consumer=consumer), self.assertRaises(ValueError): + metrics.Call(consumer, "a", "", "m") + + def test_only_a_2xx_reply_is_ok(self) -> None: + for status in (301, 302, 400, 500): + with self.subTest(status=status): + with metrics.Call("scene", str(status), "https://h/v1", "m") as call: + call.finish(Response(status, {})) + self.assertEqual([(r["status"], r["http_status"]) for r in self.records()], [ + ("http_error", 301), ("http_error", 302), + ("http_error", 400), ("http_error", 500), + ]) + + def test_a_name_utf8_cannot_encode_still_records_the_usage(self) -> None: + # A provider may answer with a lone surrogate; the record must survive it. + body = json.loads('{"model": "\\ud800", "usage": {"prompt_tokens": 4, "completion_tokens": 1}}') + with metrics.Call("scene", "relations", "https://h/v1", "m") as call: + call.finish(Response(200, body)) + [record] = self.records() + self.assertEqual((record["status"], record["input_tokens"]), ("ok", 4)) + + def test_host_matches_the_rust_writer(self) -> None: + cases = [ + ("https://user:k@API.example.com/v1?key=s", "api.example.com"), + ("http://[::1]:8000/v1", "[::1]:8000"), + ("127.0.0.1:9000/v1", "127.0.0.1:9000"), + ("", ""), + ] + for base_url, host in cases: + with self.subTest(base_url=base_url): + self.assertEqual(metrics._host(base_url), host) + + def test_unset_or_unwritable_directory_does_not_fail_the_call(self) -> None: + with mock.patch.dict(os.environ, {"ROBONIX_LLM_METRICS_DIR": ""}): + with metrics.Call("scene", "a", "", "m") as call: + call.finish() + blocker = Path(self.directory.name) / "file" + blocker.write_text("") + with mock.patch.dict(os.environ, {"ROBONIX_LLM_METRICS_DIR": str(blocker / "sub")}): + with metrics.Call("scene", "a", "", "m") as call: + call.finish() + self.assertEqual(os.listdir(self.directory.name), ["file"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/system/llm-metrics/Cargo.toml b/system/llm-metrics/Cargo.toml new file mode 100644 index 000000000..6a32c6578 --- /dev/null +++ b/system/llm-metrics/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: MulanPSL-2.0 +[package] +name = "robonix-llm-metrics" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "One JSON line per model request, for Robonix usage metrics." + +[dependencies] +serde.workspace = true +serde_json.workspace = true +robonix-scribe.workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/system/llm-metrics/README.md b/system/llm-metrics/README.md new file mode 100644 index 000000000..8d4611f77 --- /dev/null +++ b/system/llm-metrics/README.md @@ -0,0 +1,25 @@ +# robonix-llm-metrics + +Rust writer of the per-request model usage record described in +[`robonix_api.llm`](../../pylib/robonix-api/robonix_api/llm/README.md). Pilot +uses it for every chat-completions request. + +```rust +use robonix_llm_metrics::{Call, Status, directory_from_env}; + +let directory = directory_from_env(); // $ROBONIX_LLM_METRICS_DIR, read once +let mut call = Call::start(directory.as_deref(), "pilot", "plan", &base_url, &model, true); +// call.retry() for each resend; call.observe(&chunk) for each JSON body/chunk. +call.finish(Status::Ok, Some(200)); +drop(call); // the record is written here +``` + +A call dropped without `finish` is recorded as `cancelled`. Records go to +`/-.jsonl`, each appended with one write; a `None` +directory writes nothing, and a write failure is logged once through Scribe. +`consumer` must match `[a-z0-9_-]+`; `Call::start` panics otherwise, since it +names the file. + +```bash +cargo test -p robonix-llm-metrics +``` diff --git a/system/llm-metrics/src/lib.rs b/system/llm-metrics/src/lib.rs new file mode 100644 index 000000000..620b00c7e --- /dev/null +++ b/system/llm-metrics/src/lib.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: MulanPSL-2.0 +//! One JSON line per model request, for usage metrics and later billing. +//! +//! Rust counterpart of `robonix_api.llm.metrics`: the record and the usage +//! parsing are the same, and both run `contract/usage_cases.json`. Records go +//! to `/-.jsonl`, the directory usually coming from +//! [`directory_from_env`]; without a directory nothing is written. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use serde_json::Value; + +/// Record layout version; bump it when a field changes meaning. +pub const SCHEMA_VERSION: u32 = 1; + +/// Environment variable naming the directory records are appended to. +pub const DIRECTORY_ENV: &str = "ROBONIX_LLM_METRICS_DIR"; + +/// Serialises appends from concurrent requests in one process. +static WRITE_LOCK: Mutex<()> = Mutex::new(()); +/// Set after the first write failure so the warning is logged once. +static WARNED: AtomicBool = AtomicBool::new(false); + +/// Token counts a provider reported; `None` means it did not report that count. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub struct Usage { + /// Prompt tokens, including cached ones. + pub input_tokens: Option, + /// Generated tokens, including reasoning. + pub output_tokens: Option, + /// Prompt tokens served from the provider's cache, usually billed lower. + pub cached_input_tokens: Option, + /// Output tokens spent on hidden reasoning. + pub reasoning_tokens: Option, + /// Model the response names; behind a relay this is the model actually used. + pub provider_model: Option, +} + +impl Usage { + /// Read usage from a Chat Completions body or stream chunk. + #[must_use] + pub fn from_response(body: &Value) -> Self { + let usage = body.get("usage").filter(|usage| usage.is_object()); + let count = |value: Option<&Value>| value.and_then(Value::as_u64); + let field = |name: &str| usage.and_then(|usage| usage.get(name)); + let detail = |group: &str, name: &str| count(field(group).and_then(|g| g.get(name))); + Self { + input_tokens: count(field("prompt_tokens")), + output_tokens: count(field("completion_tokens")), + // DeepSeek reports cache hits outside prompt_tokens_details. + cached_input_tokens: detail("prompt_tokens_details", "cached_tokens") + .or_else(|| count(field("prompt_cache_hit_tokens"))), + reasoning_tokens: detail("completion_tokens_details", "reasoning_tokens"), + provider_model: body + .get("model") + .and_then(Value::as_str) + .filter(|model| !model.is_empty()) + .map(str::to_owned), + } + } + + /// Keep the counts this chunk reports; streams send usage in a late chunk. + pub fn merge(&mut self, other: Self) { + self.input_tokens = other.input_tokens.or(self.input_tokens); + self.output_tokens = other.output_tokens.or(self.output_tokens); + self.cached_input_tokens = other.cached_input_tokens.or(self.cached_input_tokens); + self.reasoning_tokens = other.reasoning_tokens.or(self.reasoning_tokens); + self.provider_model = other.provider_model.or(self.provider_model.take()); + } +} + +/// How a request ended, as recorded in `status`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Status { + /// The provider answered with a success status and, for a stream, the + /// whole reply was read. + Ok, + /// The provider answered with HTTP 4xx/5xx. + HttpError, + /// The request did not complete in time. + Timeout, + /// The caller abandoned the request, including a dropped request future. + Cancelled, + /// The connection failed, or the reply could not be read or ended early. + TransportError, +} + +/// One model request, written when it is dropped. +/// +/// A call that is never finished is recorded as `cancelled`, which is what +/// dropping an in-flight future means. +#[derive(Debug)] +pub struct Call { + /// Directory the record is appended to; `None` writes nothing. + directory: Option, + /// Fields written to the record. + record: Record, + /// When the request was sent, for `latency_ms`. + started: Instant, +} + +/// The JSON line for one request; field names match the Python writer. +#[derive(Debug, Serialize)] +struct Record { + /// Layout version, [`SCHEMA_VERSION`]. + v: u32, + /// Component that made the request, such as `pilot`. + consumer: String, + /// What the component used the model for, such as `plan`. + operation: String, + /// Endpoint host and port only; the path and credentials are dropped. + host: String, + /// Model name sent in the request. + model: String, + /// Whether the reply was streamed. + stream: bool, + /// Outcome of the request. + status: Status, + /// HTTP status, when a response arrived. + http_status: Option, + /// Requests sent, counting retries; each may be billed. + attempts: u32, + /// Reported token counts. + #[serde(flatten)] + usage: Usage, +} + +impl Call { + /// Start timing a request to `base_url` with `model`, recorded under + /// `directory`. + /// + /// # Panics + /// + /// When `consumer` is not a slug of `[a-z0-9_-]`: it names the file. + #[must_use] + pub fn start( + directory: Option<&Path>, + consumer: &str, + operation: &str, + base_url: &str, + model: &str, + stream: bool, + ) -> Self { + assert!( + is_slug(consumer), + "llm metrics consumer {consumer:?} must match [a-z0-9_-]+" + ); + Self { + directory: directory.map(Path::to_path_buf), + record: Record { + v: SCHEMA_VERSION, + consumer: consumer.to_owned(), + operation: operation.to_owned(), + host: host_of(base_url), + model: model.to_owned(), + stream, + status: Status::Cancelled, + http_status: None, + attempts: 1, + usage: Usage::default(), + }, + started: Instant::now(), + } + } + + /// Count one more request sent for this call, such as a retry. + pub fn retry(&mut self) { + self.record.attempts += 1; + } + + /// Record usage from a body or stream chunk. + pub fn observe(&mut self, body: &Value) { + self.record.usage.merge(Usage::from_response(body)); + } + + /// HTTP status of the last answer, when one arrived. + #[must_use] + pub fn http_status(&self) -> Option { + self.record.http_status + } + + /// Set the outcome; the record is written when the call is dropped. + pub fn finish(&mut self, status: Status, http_status: Option) { + self.record.status = status; + self.record.http_status = http_status; + } +} + +impl Drop for Call { + fn drop(&mut self) { + let Some(directory) = &self.directory else { + return; + }; + let Ok(mut line) = serde_json::to_value(&self.record) else { + return; + }; + let latency_ms = u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let ts_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }); + let boot_id = std::env::var("RBNX_BOOT_ID") + .ok() + .filter(|id| !id.is_empty()); + line["latency_ms"] = latency_ms.into(); + line["ts_ms"] = ts_ms.into(); + line["boot_id"] = boot_id.into(); + line["pid"] = std::process::id().into(); + write(directory, &self.record.consumer, &line); + } +} + +/// The directory named by [`DIRECTORY_ENV`], when it is set and not empty. +#[must_use] +pub fn directory_from_env() -> Option { + std::env::var_os(DIRECTORY_ENV) + .filter(|directory| !directory.is_empty()) + .map(PathBuf::from) +} + +fn is_slug(consumer: &str) -> bool { + !consumer.is_empty() + && consumer + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"_-".contains(&byte)) +} + +/// Endpoint host with an explicit port, without scheme, credentials or path. +fn host_of(base_url: &str) -> String { + let rest = base_url + .split_once("://") + .map_or(base_url, |(_, rest)| rest); + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host); + host.to_ascii_lowercase() +} + +/// Append one line; a metrics failure must not fail the model call. +fn write(directory: &Path, consumer: &str, line: &Value) { + let path = directory.join(format!("{consumer}-{}.jsonl", std::process::id())); + // One write of the whole line: `Value`'s Display issues many small writes, + // and a reader or a crash between them would see half a record. + let mut bytes = line.to_string().into_bytes(); + bytes.push(b'\n'); + let _guard = WRITE_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let result = std::fs::create_dir_all(directory).and_then(|()| { + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path)? + .write_all(&bytes) + }); + if let Err(error) = result + && !WARNED.swap(true, Ordering::Relaxed) + { + robonix_scribe::warn!("llm metrics: cannot write {}: {error}", path.display()); + } +} + +#[cfg(test)] +mod tests { + use super::{host_of, is_slug}; + + #[test] + fn consumer_must_be_a_file_name_slug() { + assert!(is_slug("vlm_verifier")); + assert!(!is_slug("")); + assert!(!is_slug("../pilot")); + assert!(!is_slug("Pilot")); + } + + #[test] + fn host_drops_scheme_credentials_and_path() { + assert_eq!( + host_of("https://u:k@API.example.com/v1?key=s"), + "api.example.com" + ); + assert_eq!(host_of("http://127.0.0.1:8000/v1"), "127.0.0.1:8000"); + assert_eq!(host_of(""), ""); + } +} diff --git a/system/llm-metrics/tests/contract.rs b/system/llm-metrics/tests/contract.rs new file mode 100644 index 000000000..0a7c2fd42 --- /dev/null +++ b/system/llm-metrics/tests/contract.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MulanPSL-2.0 +//! Runs the Python package's usage cases and checks the written record. + +use std::path::Path; + +use robonix_llm_metrics::{Call, Status, Usage}; +use serde_json::Value; + +/// The cases live in the Python package so both writers read one file; the +/// crate is therefore testable inside the monorepo only. +#[test] +fn usage_matches_the_shared_cases() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../pylib/robonix-api/robonix_api/llm/contract/usage_cases.json"); + let fixture: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + for case in fixture["cases"].as_array().unwrap() { + let usage = serde_json::to_value(Usage::from_response(&case["body"])).unwrap(); + assert_eq!(usage, case["expected"], "{}", case["name"]); + } +} + +#[test] +fn a_call_writes_one_record_with_the_last_reported_usage() { + let directory = tempfile::tempdir().unwrap(); + let path = Some(directory.path()); + { + let mut call = Call::start( + path, + "pilot", + "plan", + "https://k@api.example.com/v1", + "m", + true, + ); + call.retry(); + call.observe(&serde_json::json!({"model": "m-1", "choices": []})); + call.observe(&serde_json::json!({"usage": {"prompt_tokens": 9, "completion_tokens": 3}})); + call.finish(Status::Ok, Some(200)); + } + drop(Call::start(path, "pilot", "summary", "", "m", true)); + drop(Call::start(None, "pilot", "unrecorded", "", "m", true)); + let text = std::fs::read_to_string( + directory + .path() + .join(format!("pilot-{}.jsonl", std::process::id())), + ) + .unwrap(); + let records: Vec = text + .lines() + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + assert_eq!(records.len(), 2); + let first = &records[0]; + assert_eq!(first["host"], "api.example.com"); + assert_eq!( + (first["status"].as_str(), first["attempts"].as_u64()), + (Some("ok"), Some(2)) + ); + assert_eq!( + ( + first["input_tokens"].as_u64(), + first["output_tokens"].as_u64() + ), + (Some(9), Some(3)) + ); + assert_eq!(first["provider_model"], "m-1"); + assert_eq!(records[1]["status"], "cancelled"); +} From 303d7d8d848b8c78ae1c6a624678ed6f3f69fcdb Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 11:56:58 +0800 Subject: [PATCH 02/14] feat(pilot): record model usage and real outcomes for every VLM request --- Cargo.lock | 1 + system/pilot/Cargo.toml | 1 + system/pilot/README.md | 2 +- system/pilot/src/main.rs | 8 + system/pilot/src/planner.rs | 21 +- system/pilot/src/vlm.rs | 585 ++++++++++++++++++++++++++++-------- 6 files changed, 486 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 749290213..6dad02068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3337,6 +3337,7 @@ dependencies = [ "robonix-atlas", "robonix-codegen", "robonix-llm-catalog", + "robonix-llm-metrics", "robonix-scribe", "serde", "serde_json", diff --git a/system/pilot/Cargo.toml b/system/pilot/Cargo.toml index 77f0297d7..59bde54f7 100644 --- a/system/pilot/Cargo.toml +++ b/system/pilot/Cargo.toml @@ -21,6 +21,7 @@ serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } serde_yaml.workspace = true robonix-scribe.workspace = true +robonix-llm-metrics.workspace = true uuid.workspace = true rmcp = { workspace = true, features = ["client", "transport-child-process", "transport-streamable-http-client-reqwest"] } async-openai = { workspace = true, features = ["chat-completion"] } diff --git a/system/pilot/README.md b/system/pilot/README.md index f1a52bc84..620edb5f5 100644 --- a/system/pilot/README.md +++ b/system/pilot/README.md @@ -141,7 +141,7 @@ Both Soma blocks are projected onto what a planner acts on. From the body descri The catalog prints each capability's name, the opening paragraph of its description, and its argument schema. A description longer than that paragraph — a provider that wrote its request/response manual into the description field rather than a `CAPABILITY.md` — is summarized, and the entry carries a `more:` line naming the `provider_id` to pass to the Executor's `read_capability_doc` builtin for the full text. The catalog stays in history, with later rounds appending only changes. A manual is fetched when the model requests it. -Every request writes a `[pilot/prompt]` JSON log with text bytes and four-byte token estimates for standing context, RTDL protocol, current runtime-context sections (including the capability catalog record, when there is one), and full ordered history. Pilot supplies one opaque `prompt_cache_key` for the **lifetime of the running Pilot deployment**, rather than one key per task: the unchanged system prefix and completed histories can route to the same cache across independent tasks without using a user/session identifier. The message-ordering test asserts that a later normal planning request is an exact extension of the prior wire-message sequence; this is the code-level reusable-prefix invariant. Compatible providers also report exact input, output, total, cached-input, and uncached-input tokens through streaming usage. Pilot logs those values per request together with `cache_hit` and `cache_hit_ratio`, plus cumulative token totals, cache-hit request counts, and the cumulative cache-hit ratio for the interaction. Provider-reported cache ratio remains a runtime measurement: routing, TTL expiry, provider cache granularity, and a newly appended suffix make it distinct from code-level prefix reuse. Pricing is not hard-coded because an OpenAI-compatible proxy may route the same model name to differently priced backends; the logged totals can be priced against the operator's actual provider contract. The fake VLM's reported usage is explicitly a deterministic four-byte estimate, not a production tokenizer result. A provider that rejects a named optional cache/usage field with HTTP 400 or 422 is retried once without those fields; unrelated client errors are returned unchanged. +Every request writes a `[pilot/prompt]` JSON log with text bytes and four-byte token estimates for standing context, RTDL protocol, current runtime-context sections (including the capability catalog record, when there is one), and full ordered history. Pilot supplies one opaque `prompt_cache_key` for the **lifetime of the running Pilot deployment**, rather than one key per task: the unchanged system prefix and completed histories can route to the same cache across independent tasks without using a user/session identifier. The message-ordering test asserts that a later normal planning request is an exact extension of the prior wire-message sequence; this is the code-level reusable-prefix invariant. Compatible providers also report exact input, output, total, cached-input, and uncached-input tokens through streaming usage. Pilot logs those values per request together with `cache_hit` and `cache_hit_ratio`, plus cumulative token totals, cache-hit request counts, and the cumulative cache-hit ratio for the interaction. Provider-reported cache ratio remains a runtime measurement: routing, TTL expiry, provider cache granularity, and a newly appended suffix make it distinct from code-level prefix reuse. Pricing is not hard-coded because an OpenAI-compatible proxy may route the same model name to differently priced backends; the logged totals can be priced against the operator's actual provider contract. Each call to the provider also appends one usage record (consumer `pilot`, operation `plan` or `summary`) whose `attempts` counts the requests it sent, retries included; the planner's single reopen after a failed open is a second call with its own record. Records are written through `robonix-llm-metrics` when `ROBONIX_LLM_METRICS_DIR` is set; startup logs whether it is. A planning request that opens or idles past `ROBONIX_PILOT_VLM_IDLE_TIMEOUT_SECS` is recorded as `timeout`, and a reply that ends without `[DONE]` or a finish reason as `transport_error`. The fake VLM's reported usage is explicitly a deterministic four-byte estimate, not a production tokenizer result. A provider that rejects a named optional cache/usage field with HTTP 400 or 422 is retried once without those fields; unrelated client errors are returned unchanged. The `info` records above carry sizes and token counts, never prompt text. To see exactly what the model receives and returns, raise Pilot's log level to `debug`: set `log: debug` on the Pilot component in the deployment manifest, or `SCRIBE_FILE_LEVEL=debug` for the log file and `SCRIBE_CONSOLE_LEVEL=debug` for the terminal. Each planning request then also writes `[pilot/prompt/messages]`, the full ordered message sequence sent to the provider with inline images reduced to their byte size, and each reply writes `[pilot/rtdl/raw]`, the model output as received before parsing. These records contain user tasks and scene contents verbatim, so enable them only while debugging. diff --git a/system/pilot/src/main.rs b/system/pilot/src/main.rs index acb5c4c90..aa83924db 100644 --- a/system/pilot/src/main.rs +++ b/system/pilot/src/main.rs @@ -226,6 +226,14 @@ async fn main() -> Result<()> { cfg.vlm.endpoint_for_log(), cfg.vlm.model ); + // Without this line an unset variable loses every usage record silently. + match robonix_llm_metrics::directory_from_env() { + Some(directory) => info!("[pilot/vlm] usage records -> {}", directory.display()), + None => info!( + "[pilot/vlm] usage records off ({} unset)", + robonix_llm_metrics::DIRECTORY_ENV + ), + } info!( "[pilot/context_budget] {}", serde_json::json!({ diff --git a/system/pilot/src/planner.rs b/system/pilot/src/planner.rs index 7048234d1..6494c30c3 100644 --- a/system/pilot/src/planner.rs +++ b/system/pilot/src/planner.rs @@ -1118,7 +1118,7 @@ struct VlmTextCompletion { /// compaction ledger. Best-effort: callers keep source history if it fails. async fn collect_vlm_text(vlm: &VlmClient, messages: &[Message]) -> Option { let mut stream = vlm - .chat_stream(messages, &[], None, ReplyShape::Text) + .chat_stream(messages, &[], None, ReplyShape::Text, "summary", None) .await .ok()?; let mut text = String::new(); @@ -1635,13 +1635,16 @@ pub async fn run_turn( // Opening cannot dispatch RTDL nodes, so one retry is safe here. let mut open_retried = false; let mut stream = loop { - match tokio::time::timeout( - vlm_idle_timeout(), - vlm.chat_stream(&messages, &[], Some(vlm.prompt_cache_key()), reply_shape), - ) - .await - .map_err(anyhow::Error::from) - .and_then(|result| result) + match vlm + .chat_stream( + &messages, + &[], + Some(vlm.prompt_cache_key()), + reply_shape, + "plan", + Some(vlm_idle_timeout()), + ) + .await { Ok(stream) => break stream, Err(error) if !open_retried => { @@ -1702,7 +1705,7 @@ pub async fn run_turn( Err(error) => { live.transport_failed = true; stream_error = Some(error.into()); break; } } } - _ = &mut idle => { stream_error = Some(anyhow::anyhow!("VLM stream idle timeout")); break; } + () = &mut idle => { stream.mark_timed_out(); stream_error = Some(anyhow::anyhow!("VLM stream idle timeout")); break; } item = stream.next() => { idle.as_mut().reset(tokio::time::Instant::now() + vlm_idle_timeout()); match item { diff --git a/system/pilot/src/vlm.rs b/system/pilot/src/vlm.rs index 84f51c31b..68fb50575 100644 --- a/system/pilot/src/vlm.rs +++ b/system/pilot/src/vlm.rs @@ -16,10 +16,15 @@ use async_openai::types::chat::{ FunctionObjectArgs, ImageDetail, ImageUrl, ResponseFormat, ResponseFormatJsonSchema, }; use futures_util::stream::{Stream, StreamExt}; +use robonix_llm_metrics::{Call, Status, Usage}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; +use std::path::PathBuf; use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::{Context as TaskContext, Poll}; use std::time::Duration; use uuid::Uuid; @@ -369,6 +374,35 @@ pub struct VlmClient { /// would defeat reuse between independent tasks with the same standing /// prompt. It is opaque and is not derived from a user/session id. prompt_cache_key: String, + /// Where each request's usage record goes; `None` records nothing. + metrics_dir: Option, +} + +/// Items of one streamed completion, as returned by [`VlmClient::chat_stream`]. +/// +/// Dropping it abandons the request. [`VlmStream::mark_timed_out`] first +/// tells the usage record that the caller gave up because the stream went +/// idle, which is otherwise indistinguishable from a cancellation. +pub struct VlmStream { + /// Items forwarded by the task reading the upstream reply. + items: tokio_stream::wrappers::ReceiverStream>, + /// Set by the caller before dropping a stream it timed out. + timed_out: Arc, +} + +impl VlmStream { + /// Record the request as `timeout` rather than `cancelled` once dropped. + pub fn mark_timed_out(&self) { + self.timed_out.store(true, Ordering::Release); + } +} + +impl Stream for VlmStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + Pin::new(&mut self.items).poll_next(cx) + } } /// What the caller wants back from one completion. @@ -566,6 +600,7 @@ impl VlmClient { reasoning_effort: cfg.reasoning_effort.clone(), configured_context_window_tokens: cfg.context_window_tokens, prompt_cache_key: Uuid::new_v4().simple().to_string(), + metrics_dir: robonix_llm_metrics::directory_from_env(), } } @@ -657,13 +692,17 @@ impl VlmClient { /// - `ToolCall` once per accumulated function call (after the upstream /// finishes streaming all argument deltas) /// - one final `Finish` + /// + /// `open_timeout` bounds opening the stream, retries included. pub async fn chat_stream( &self, messages: &[Message], tools: &[ToolDef], prompt_cache_key: Option<&str>, reply_shape: ReplyShape, - ) -> Result> + Send>>> { + operation: &str, + open_timeout: Option, + ) -> Result { let oai_messages = build_openai_messages(messages)?; let oai_tools = build_openai_tools(tools)?; @@ -712,100 +751,150 @@ impl VlmClient { // Optional fields go on the first rejection; the response format can // need a second step (see downgrade_response_format). let mut compatibility_fallbacks = 0_u8; - let response = loop { - let mut request = self.inner.post(&url); - if !self.api_key.is_empty() { - request = request.bearer_auth(&self.api_key); - } - let response = request - .header(reqwest::header::ACCEPT, "text/event-stream") - .header(reqwest::header::CONTENT_TYPE, "application/json") - .json(&request_body) - .send() - .await - .context("open VLM chat stream")?; - let status = response.status(); - if status.is_success() { - break response; - } - let retry_after = response - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|value| value.to_str().ok()) - .map(str::to_string); - let text = response.text().await.unwrap_or_default(); - // Providers word a refused response_format in their own way - // (MiniMax: "Mismatch type string with value array"), so any 400 or - // 422 to a request carrying one steps it down; an unrelated error - // costs one more request and is reported as it was. - let format_refused = matches!(status.as_u16(), 400 | 422) - && request_body.get("response_format").is_some(); - if (format_refused || rejects_optional_request_fields(status, &text)) - && compatibility_fallbacks < 2 - { - // Drop an optional field only when the error names it: removing - // the cache key with a refused format would forgo prompt - // caching, and stream_options the usage report, on every - // later request of the stream. - let lowered = text.to_ascii_lowercase(); - let names_format = - lowered.contains("response_format") || lowered.contains("json_schema"); - let downgraded = if format_refused || names_format { - downgrade_response_format(&mut request_body) - } else { - None + let mut call = Call::start( + self.metrics_dir.as_deref(), + "pilot", + operation, + &self.api_base, + &self.model, + true, + ); + let opened = async { + let opened = loop { + let mut request = self.inner.post(&url); + if !self.api_key.is_empty() { + request = request.bearer_auth(&self.api_key); + } + let sent = request + .header(reqwest::header::ACCEPT, "text/event-stream") + .header(reqwest::header::CONTENT_TYPE, "application/json") + .json(&request_body) + .send() + .await; + let response = match sent { + Ok(response) => response, + Err(error) => { + call.finish(Status::TransportError, None); + return Err(error).context("open VLM chat stream"); + } + }; }; - let mut dropped: Vec<&str> = Vec::new(); - if let Some(body) = request_body.as_object_mut() { - for (field, named) in [ - ("stream_options", "stream_options"), - ("stream_options", "include_usage"), - ("prompt_cache_key", "prompt_cache_key"), - ("prompt_cache_options", "prompt_cache_options"), - ] { - if lowered.contains(named) && body.remove(field).is_some() { - dropped.push(field); + let status = response.status(); + if status.is_success() { + // Until the stream ends, dropping it means the caller gave up. + call.finish(Status::Cancelled, Some(status.as_u16())); + break (response, status.as_u16()); + } + call.finish(Status::HttpError, Some(status.as_u16())); + let retry_after = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let text = response.text().await.unwrap_or_default(); + // Providers word a refused response_format in their own way + // (MiniMax: "Mismatch type string with value array"), so any 400 or + // 422 to a request carrying one steps it down; an unrelated error + // costs one more request and is reported as it was. + let format_refused = matches!(status.as_u16(), 400 | 422) + && request_body.get("response_format").is_some(); + if (format_refused || rejects_optional_request_fields(status, &text)) + && compatibility_fallbacks < 2 + { + // Drop an optional field only when the error names it: removing + // the cache key with a refused format would forgo prompt + // caching, and stream_options the usage report, on every + // later request of the stream. + let lowered = text.to_ascii_lowercase(); + let names_format = + lowered.contains("response_format") || lowered.contains("json_schema"); + let downgraded = if format_refused || names_format { + downgrade_response_format(&mut request_body) + } else { + None + }; + let mut dropped: Vec<&str> = Vec::new(); + if let Some(body) = request_body.as_object_mut() { + for (field, named) in [ + ("stream_options", "stream_options"), + ("stream_options", "include_usage"), + ("prompt_cache_key", "prompt_cache_key"), + ("prompt_cache_options", "prompt_cache_options"), + ] { + if lowered.contains(named) && body.remove(field).is_some() { + dropped.push(field); + } } } - } - if dropped.is_empty() && downgraded.is_none() { - bail!("open VLM chat stream: HTTP {status}: {text}"); - } - // Name what changed rather than "retrying without them": a - // downgrade is not a removal, and a run that later reads as - // schema-guided needs this line to say it was not. - if let Some(now) = downgraded { - robonix_scribe::warn!( - "[pilot/vlm] upstream rejected response_format with HTTP {status}; \ + if dropped.is_empty() && downgraded.is_none() { + bail!("open VLM chat stream: HTTP {status}: {text}"); + } + // Name what changed rather than "retrying without them": a + // downgrade is not a removal, and a run that later reads as + // schema-guided needs this line to say it was not. + if let Some(now) = downgraded { + robonix_scribe::warn!( + "[pilot/vlm] upstream rejected response_format with HTTP {status}; \ response_format is now {now} for the rest of this stream. RTDL shape \ is no longer schema-guided; admission still runs in Pilot. \ Upstream said: {text}" - ); + ); + } + if !dropped.is_empty() { + robonix_scribe::warn!( + "[pilot/vlm] upstream rejected optional request fields with HTTP {status}; \ + retrying without {}", + dropped.join(", ") + ); + } + compatibility_fallbacks += 1; + call.retry(); + continue; } - if !dropped.is_empty() { + if let Some(delay) = open_retry_delay(status, retry_after.as_deref(), retry_index) { robonix_scribe::warn!( - "[pilot/vlm] upstream rejected optional request fields with HTTP {status}; \ - retrying without {}", - dropped.join(", ") + "[pilot/vlm] open stream HTTP {status}; retry {}/{} in {:.1}s", + retry_index + 1, + MAX_OPEN_RETRIES, + delay.as_secs_f64() ); + tokio::time::sleep(delay).await; + retry_index += 1; + call.retry(); + continue; } - compatibility_fallbacks += 1; - continue; - } - if let Some(delay) = open_retry_delay(status, retry_after.as_deref(), retry_index) { - robonix_scribe::warn!( - "[pilot/vlm] open stream HTTP {status}; retry {}/{} in {:.1}s", - retry_index + 1, - MAX_OPEN_RETRIES, - delay.as_secs_f64() - ); - tokio::time::sleep(delay).await; - retry_index += 1; - continue; - } - bail!("open VLM chat stream: HTTP {status}: {text}"); + bail!("open VLM chat stream: HTTP {status}: {text}"); + }; + anyhow::Ok(opened) }; + let opened = match open_timeout { + Some(limit) => match tokio::time::timeout(limit, opened).await { + Ok(opened) => opened, + Err(_) => { + // Keep a 429 seen before a long Retry-After wait. + let last = call.http_status(); + call.finish(Status::Timeout, last); + bail!( + "open VLM chat stream: no response within {:.1}s", + limit.as_secs_f64() + ); + } + }, + None => opened.await, + }; + let (response, http_status) = opened?; let mut upstream = response.bytes_stream(); + let timed_out = Arc::new(AtomicBool::new(false)); + let abandoned = { + let timed_out = Arc::clone(&timed_out); + move || { + if timed_out.load(Ordering::Acquire) { + Status::Timeout + } else { + Status::Cancelled + } + } + }; // Walk the upstream chunk-by-chunk, accumulating tool-call deltas by // index until the upstream finishes; then emit one ToolCall per index @@ -814,14 +903,18 @@ impl VlmClient { let (tx, rx) = tokio::sync::mpsc::channel::>(64); tokio::spawn(async move { let mut tc_acc: BTreeMap = BTreeMap::new(); - let mut finish = "stop".to_string(); + // Stays empty unless the upstream names a finish reason. + let mut finish = String::new(); // HTTP chunks may end inside a UTF-8 character. Decode only after // a complete SSE line arrives (or the transport reaches EOF). let mut buf = Vec::new(); let mut done = false; while !done { let chunk = tokio::select! { - _ = tx.closed() => return, + () = tx.closed() => { + call.finish(abandoned(), Some(http_status)); + return; + } chunk = upstream.next() => chunk, }; let Some(chunk) = chunk else { @@ -832,13 +925,26 @@ impl VlmClient { buf.extend_from_slice(&bytes); while let Some(pos) = buf.iter().position(|byte| *byte == b'\n') { let line: Vec = buf.drain(..=pos).collect(); - match process_stream_line(&line, &mut tc_acc, &mut finish, &tx).await { - Ok(true) => { + match process_stream_line( + &line, + &mut tc_acc, + &mut finish, + &tx, + &mut call, + ) + .await + { + Ok(Some(StreamEnd::Complete)) => { done = true; break; } - Ok(false) => {} + Ok(Some(StreamEnd::Abandoned)) => { + call.finish(abandoned(), Some(http_status)); + return; + } + Ok(None) => {} Err(e) => { + call.finish(Status::TransportError, Some(http_status)); let _ = tx.send(Err(e)).await; return; } @@ -846,6 +952,7 @@ impl VlmClient { } } Err(e) => { + call.finish(Status::TransportError, Some(http_status)); let _ = tx .send(Err(anyhow::anyhow!("VLM stream chunk error: {e}"))) .await; @@ -853,13 +960,34 @@ impl VlmClient { } } } - if !done - && !buf.is_empty() - && let Err(e) = process_stream_line(&buf, &mut tc_acc, &mut finish, &tx).await - { - let _ = tx.send(Err(e)).await; - return; + if !done && !buf.is_empty() { + match process_stream_line(&buf, &mut tc_acc, &mut finish, &tx, &mut call).await { + Ok(Some(StreamEnd::Abandoned)) => { + call.finish(abandoned(), Some(http_status)); + return; + } + Ok(Some(StreamEnd::Complete)) => done = true, + Ok(None) => {} + Err(e) => { + call.finish(Status::TransportError, Some(http_status)); + let _ = tx.send(Err(e)).await; + return; + } + } } + // A reply that ends without [DONE] or a finish reason was cut off; + // it is still passed on, but not counted as a complete answer. A + // consumer that stops reading the remaining items does not make a + // complete reply cancelled. + let complete = done || !finish.is_empty(); + call.finish( + if complete { + Status::Ok + } else { + Status::TransportError + }, + Some(http_status), + ); for (_, tc) in tc_acc { if tc.id.is_empty() && tc.name.is_empty() { @@ -881,7 +1009,10 @@ impl VlmClient { let _ = tx.send(Ok(VlmStreamItem::Finish)).await; }); - Ok(Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx))) + Ok(VlmStream { + items: tokio_stream::wrappers::ReceiverStream::new(rx), + timed_out, + }) } } @@ -914,6 +1045,7 @@ mod tests { downgrade_response_format, open_retry_delay, parse_usage, process_stream_line, rejects_optional_request_fields, response_format_for, }; + use robonix_llm_metrics::Call; use serde_json::json; use std::collections::BTreeMap; use std::time::Duration; @@ -1130,6 +1262,7 @@ mod tests { &mut calls, &mut finish, &tx, + &mut Call::start(None, "test", "test", "", "", true), ) .await .unwrap(); @@ -1151,6 +1284,7 @@ mod tests { &mut calls, &mut finish, &tx, + &mut Call::start(None, "test", "test", "", "", true), ) .await .unwrap(); @@ -1233,9 +1367,17 @@ mod tests { reasoning_effort: effort.map(str::to_owned), configured_context_window_tokens: None, prompt_cache_key: "local-test".into(), + metrics_dir: None, }; let stream = client - .chat_stream(&[super::Message::user("test")], &[], None, ReplyShape::Text) + .chat_stream( + &[super::Message::user("test")], + &[], + None, + ReplyShape::Text, + "test", + None, + ) .await .unwrap(); let items = tokio::time::timeout(Duration::from_secs(10), stream.collect::>()) @@ -1296,6 +1438,214 @@ mod tests { assert!(matches!(&items[0], Ok(VlmStreamItem::TextDelta(value)) if value == "中文")); // i18n-ok: multibyte UTF-8 test fixture assert!(matches!(&items[1], Ok(VlmStreamItem::Finish))); } + + /// Answer one request with a canned HTTP reply, holding the socket open + /// when `hold` so the client keeps waiting for more of the body. + async fn canned(response: Vec, hold: bool) -> (String, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut buf = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buf).await.unwrap(); + assert!(count > 0, "client closed before sending its request"); + request.extend_from_slice(&buf[..count]); + } + socket.write_all(&response).await.unwrap(); + if hold { + tokio::time::sleep(Duration::from_secs(30)).await; + } + }); + (format!("http://{address}"), handle) + } + + /// A canned HTTP reply carrying `body` as the whole SSE stream. + fn chat_response(status: &str, body: &str) -> Vec { + format!( + "HTTP/1.1 {status}\r\nContent-Type: text/event-stream\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .into_bytes() + } + + fn test_client(base: &str, metrics_dir: &std::path::Path) -> super::VlmClient { + super::VlmClient { + inner: reqwest::Client::builder().no_proxy().build().unwrap(), + api_base: base.to_string(), + api_key: "local-test".into(), + model: "local-test".into(), + configured_context_window_tokens: None, + prompt_cache_key: "local-test".into(), + metrics_dir: Some(metrics_dir.to_path_buf()), + } + } + + /// Open a text stream for `operation` against `base`. + async fn open( + base: &str, + metrics_dir: &std::path::Path, + operation: &str, + open_timeout: Option, + ) -> anyhow::Result { + test_client(base, metrics_dir) + .chat_stream( + &[super::Message::user("test")], + &[], + None, + ReplyShape::Text, + operation, + open_timeout, + ) + .await + } + + /// Records written under `directory` for `operation`, waiting for the + /// background reader to write one when `wait` is set. + async fn records( + directory: &std::path::Path, + operation: &str, + wait: bool, + ) -> Vec { + let path = directory.join(format!("pilot-{}.jsonl", std::process::id())); + for _ in 0..100 { + let text = std::fs::read_to_string(&path).unwrap_or_default(); + let found: Vec = text + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|record| record["operation"] == operation) + .collect(); + if !found.is_empty() || !wait { + return found; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + Vec::new() + } + + /// Every exit path writes one record with the outcome it really had. + #[tokio::test] + async fn chat_stream_records_each_outcome() { + use futures_util::StreamExt; + + let directory = tempfile::tempdir().unwrap(); + let dir = directory.path(); + + let complete = format!( + "data: {}\n\ndata: [DONE]\n\n", + json!({ + "choices": [{"delta": {"content": "hi"}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}, + }) + ); + let (base, server) = canned(chat_response("200 OK", &complete), false).await; + let stream = open(&base, dir, "done", None).await.unwrap(); + let _: Vec<_> = stream.collect().await; + server.await.unwrap(); + let [record] = records(dir, "done", true).await.try_into().unwrap(); + assert_eq!(record["status"], "ok"); + assert_eq!(record["input_tokens"], 3); + assert_eq!(record["attempts"], 1); + + // The connection closes before [DONE] or a finish reason. + let cut = "data: {\"choices\":[{\"delta\":{\"content\":\"h\"}}]}\n\n"; + let (base, server) = canned(chat_response("200 OK", cut), false).await; + let stream = open(&base, dir, "cut", None).await.unwrap(); + let _: Vec<_> = stream.collect().await; + server.await.unwrap(); + let [record] = records(dir, "cut", true).await.try_into().unwrap(); + assert_eq!( + (record["status"].as_str(), record["http_status"].as_u64()), + (Some("transport_error"), Some(200)) + ); + + let rejected = chat_response("400 Bad Request", r#"{"error":"bad model"}"#); + let (base, server) = canned(rejected, false).await; + let error = open(&base, dir, "rejected", None) + .await + .err() + .expect("a 400 must fail the request"); + assert!(format!("{error:#}").contains("400"), "{error:#}"); + server.await.unwrap(); + let [record] = records(dir, "rejected", false).await.try_into().unwrap(); + assert_eq!( + (record["status"].as_str(), record["http_status"].as_u64()), + (Some("http_error"), Some(400)) + ); + + // Content-Length larger than the body keeps the reply in flight. + let pending = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + "data: {\"choices\":[]}\n\n".len() + 4096, + "data: {\"choices\":[]}\n\n" + ); + for (operation, idle) in [("abandoned", false), ("idle", true)] { + let (base, server) = canned(pending.clone().into_bytes(), true).await; + let stream = open(&base, dir, operation, None).await.unwrap(); + if idle { + stream.mark_timed_out(); + } + drop(stream); + let [record] = records(dir, operation, true).await.try_into().unwrap(); + server.abort(); + let expected = if idle { "timeout" } else { "cancelled" }; + assert_eq!(record["status"], expected, "{operation}"); + } + + // A refused optional field is dropped and the request sent again. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let replies = [ + chat_response("400 Bad Request", r#"{"error":"unknown stream_options"}"#), + chat_response("200 OK", &complete), + ]; + let server = tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for reply in replies { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut buf = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buf).await.unwrap(); + request.extend_from_slice(&buf[..count]); + } + socket.write_all(&reply).await.unwrap(); + } + }); + let stream = open(&base, dir, "fallback", None).await.unwrap(); + let _: Vec<_> = stream.collect().await; + server.await.unwrap(); + let [record] = records(dir, "fallback", true).await.try_into().unwrap(); + assert_eq!( + (record["status"].as_str(), record["attempts"].as_u64()), + (Some("ok"), Some(2)) + ); + + // No response headers at all: opening times out. + let silent = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", silent.local_addr().unwrap()); + let error = open(&base, dir, "silent", Some(Duration::from_millis(200))) + .await + .err() + .expect("a silent upstream must time out"); + assert!(format!("{error:#}").contains("no response"), "{error:#}"); + let [record] = records(dir, "silent", false).await.try_into().unwrap(); + assert_eq!(record["status"], "timeout"); + drop(silent); + } +} + +/// How the stream walk ended, as far as the record is concerned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StreamEnd { + /// The upstream finished the reply; the record can be `ok`. + Complete, + /// The consumer stopped reading, so the reply was abandoned. + Abandoned, } #[derive(Default)] @@ -1312,34 +1662,36 @@ async fn process_stream_line( tc_acc: &mut BTreeMap, finish: &mut String, tx: &tokio::sync::mpsc::Sender>, -) -> Result { + call: &mut Call, +) -> Result> { let line = std::str::from_utf8(line) .context("VLM SSE line contains invalid UTF-8")? .trim(); if line.is_empty() || line.starts_with(':') { - return Ok(false); + return Ok(None); } let Some(data) = line.strip_prefix("data:") else { - return Ok(false); + return Ok(None); }; let data = data.trim(); if data == "[DONE]" { - return Ok(true); + return Ok(Some(StreamEnd::Complete)); } let v: Value = serde_json::from_str(data) .with_context(|| format!("deserialize VLM stream chunk: {data}"))?; + call.observe(&v); if let Some(usage) = parse_usage(&v) && tx.send(Ok(VlmStreamItem::Usage(usage))).await.is_err() { - return Ok(true); + return Ok(Some(StreamEnd::Abandoned)); } let Some(choice) = v .get("choices") .and_then(Value::as_array) .and_then(|choices| choices.first()) else { - return Ok(false); + return Ok(None); }; if let Some(content) = choice @@ -1352,7 +1704,7 @@ async fn process_stream_line( .await .is_err() { - return Ok(true); + return Ok(Some(StreamEnd::Abandoned)); } // A thinking model streams this for as long as it reasons, with no // `content` in sight. Forward it so the planner's idle timer measures the @@ -1373,7 +1725,7 @@ async fn process_stream_line( .await .is_err() { - return Ok(true); + return Ok(Some(StreamEnd::Abandoned)); } if let Some(tc_chunks) = choice .get("delta") @@ -1399,28 +1751,17 @@ async fn process_stream_line( if let Some(fr) = choice.get("finish_reason").and_then(Value::as_str) { *finish = fr.to_string(); } - Ok(false) + Ok(None) } -/// Read the standard Chat Completions usage shape without requiring every -/// OpenAI-compatible provider to deserialize optional detail fields equally. +/// The chunk's usage when it reports both prompt and completion counts. fn parse_usage(value: &Value) -> Option { - let usage = value.get("usage")?.as_object()?; - let prompt_tokens = usage.get("prompt_tokens")?.as_u64()?; - let completion_tokens = usage.get("completion_tokens")?.as_u64()?; - let cached_tokens = usage - .get("prompt_tokens_details") - .and_then(Value::as_object) - .and_then(|details| details.get("cached_tokens")) - .and_then(Value::as_u64); + let usage = Usage::from_response(value); Some(VlmUsage { - prompt_tokens, - completion_tokens, - cached_tokens, - model: value - .get("model") - .and_then(Value::as_str) - .map(str::to_string), + prompt_tokens: usage.input_tokens?, + completion_tokens: usage.output_tokens?, + cached_tokens: usage.cached_input_tokens, + model: usage.provider_model, }) } From 31ab726b907b98d5709da424ba3c2be7bad203ec Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 11:56:58 +0800 Subject: [PATCH 03/14] feat(services): record model usage in Scene, Memory, Memsearch and VLM Verifier --- .../memory/memory_service/core/llm_search.py | 7 +- .../memory/memory_service/core/observe.py | 14 ++-- .../memsearch/memsearch_service/service.py | 41 +++++++++-- services/memsearch/pyproject.toml | 5 +- services/memsearch/tests/test_lifecycle.py | 71 +++++++++++++++++++ .../vlm_verifier/vlm_verifier/vlm.py | 24 ++++--- .../scene_service/ingest/perception_vlm.py | 7 +- system/scene/scene_service/object_captions.py | 3 +- .../scene_graph/image_relations.py | 1 + .../scene_service/scene_graph/llm_client.py | 15 ++-- .../scene_service/scene_graph/relations.py | 1 + system/scene/scripts/start.sh | 9 +++ uv.lock | 4 +- 13 files changed, 172 insertions(+), 30 deletions(-) diff --git a/services/memory/memory_service/core/llm_search.py b/services/memory/memory_service/core/llm_search.py index 0346638bc..bcb3b21f4 100644 --- a/services/memory/memory_service/core/llm_search.py +++ b/services/memory/memory_service/core/llm_search.py @@ -174,6 +174,7 @@ async def _call_llm(prompt: str, max_tokens: int = 512) -> Optional[List[int]]: return None import httpx + from robonix_api.llm import metrics url = f"{cfg['base_url'].rstrip('/')}/chat/completions" headers = {"Content-Type": "application/json"} @@ -188,8 +189,10 @@ async def _call_llm(prompt: str, max_tokens: int = 512) -> Optional[List[int]]: t0 = time.monotonic() try: - async with httpx.AsyncClient(timeout=30.0) as client: - r = await client.post(url, json=body, headers=headers) + with metrics.Call("memory", "search", cfg["base_url"], cfg["model"]) as call: + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post(url, json=body, headers=headers) + call.finish(r) if r.status_code >= 400: log.warning("llm_search: LLM returned %d: %s", r.status_code, r.text[:200]) diff --git a/services/memory/memory_service/core/observe.py b/services/memory/memory_service/core/observe.py index 77c2638a3..5de66fb9e 100644 --- a/services/memory/memory_service/core/observe.py +++ b/services/memory/memory_service/core/observe.py @@ -88,6 +88,7 @@ async def _vlm_describe( return None import httpx + from robonix_api.llm import metrics # Build image content part img_b64 = base64.b64encode(image_bytes).decode("ascii") @@ -114,8 +115,10 @@ async def _vlm_describe( } try: - async with httpx.AsyncClient(timeout=30.0) as client: - r = await client.post(url, json=body, headers=headers) + with metrics.Call("memory", "observe", cfg["base_url"], cfg["model"]) as call: + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post(url, json=body, headers=headers) + call.finish(r) if r.status_code >= 400: log.warning("observe: VLM returned %d: %s", r.status_code, r.text[:200]) return None @@ -170,6 +173,7 @@ async def vlm_answer_question( return None import httpx + from robonix_api.llm import metrics # Load images from disk content_parts: List[Dict[str, Any]] = [] @@ -223,8 +227,10 @@ async def vlm_answer_question( } try: - async with httpx.AsyncClient(timeout=30.0) as client: - r = await client.post(url, json=body, headers=headers) + with metrics.Call("memory", "answer", cfg["base_url"], cfg["model"]) as call: + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post(url, json=body, headers=headers) + call.finish(r) if r.status_code >= 400: log.warning("vlm_qa: VLM returned %d: %s", r.status_code, r.text[:200]) return None diff --git a/services/memsearch/memsearch_service/service.py b/services/memsearch/memsearch_service/service.py index 22cbc12c2..7f3e32936 100644 --- a/services/memsearch/memsearch_service/service.py +++ b/services/memsearch/memsearch_service/service.py @@ -266,6 +266,40 @@ def _compact_config() -> dict[str, str | None]: "requires_key": True, } +async def _compact(base_url: str, api_key: str, model: str) -> Path | None: + """Summarize every indexed chunk into today's memory file and index it. + + Mirrors ``MemSearch.compact`` but sends the request itself, because + memsearch's own request returns no usage for the metrics record. Unlike + the openai SDK it replaced, it does not retry: the caller can compact again.""" + import httpx # noqa: PLC0415 + from memsearch.compact import COMPACT_PROMPT # noqa: PLC0415 + from robonix_api.llm import metrics # noqa: PLC0415 + + backend = _require_backend() + chunks = backend.store.query(filter_expr="") + if not chunks: + return None + prompt = COMPACT_PROMPT.format(chunks="\n\n---\n\n".join(c["content"] for c in chunks)) + with metrics.Call("memsearch", "compact", base_url, model) as call: + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post( + f"{base_url.rstrip('/')}/chat/completions", + headers={"Authorization": f"Bearer {api_key}"}, + json={"model": model, "messages": [{"role": "user", "content": prompt}]}, + ) + call.finish(response) + response.raise_for_status() + summary = response.json()["choices"][0]["message"]["content"] or "" + path = Path(MEMORY_DIR) / "memory" / f"{date.today()}.md" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as sink: + if sink.tell() == 0: + sink.write(f"# {date.today()}\n") + sink.write(f"\n\n## Memory Compact\n\n{summary}\n") + await backend.index_file(path) + return path + @memory.mcp("robonix/service/memory/compact") async def compact(msg: Empty) -> String: @@ -286,10 +320,9 @@ async def compact(msg: Empty) -> String: return String(data="compact: no LLM credentials available. " "Set VLM_API_KEY in the deploy manifest's system.memory env block.") try: - summary_path = await _require_backend().compact( - llm_provider="openai", model=model, - base_url=base_url, api_key=api_key, - ) + summary_path = await _compact(base_url or "https://api.openai.com/v1", api_key, model) + if summary_path is None: + return String(data="Nothing to compact: memory is empty.") log.info("compact done → %s", summary_path) return String(data=f"Memory compacted successfully to {summary_path}.") except Exception as e: diff --git a/services/memsearch/pyproject.toml b/services/memsearch/pyproject.toml index 90b3ff15b..31e683ff4 100644 --- a/services/memsearch/pyproject.toml +++ b/services/memsearch/pyproject.toml @@ -17,7 +17,10 @@ dependencies = [ # path, and every caller still imports mcp.server.fastmcp. "mcp>=1.27,<2", "fastmcp>=3,<5", - "memsearch[onnx]", + # compact sends its own request so the usage record gets token counts. + "httpx", + # service.py imports memsearch.compact.COMPACT_PROMPT, an internal name. + "memsearch[onnx]>=0.4.11,<0.5", # Newer official ARM64 wheels abort on Jetson power modes with offline # cores. 1.18.1 plus our explicit SessionOptions path is verified here. "onnxruntime==1.18.1; platform_machine == 'aarch64'", diff --git a/services/memsearch/tests/test_lifecycle.py b/services/memsearch/tests/test_lifecycle.py index ec9e23537..e37eaef54 100644 --- a/services/memsearch/tests/test_lifecycle.py +++ b/services/memsearch/tests/test_lifecycle.py @@ -278,6 +278,77 @@ def test_catalog_errors_fail_before_backend_construction(self) -> None: self.assertIsNone(module.mem) self.assertEqual(_WorkingMemSearch.instances, []) + def test_compact_records_the_usage_of_its_request(self) -> None: + import asyncio + import json + + sys.path.insert(0, str(PACKAGE_ROOT.parents[1] / "pylib" / "robonix-api")) + self.addCleanup(sys.path.remove, sys.path[0]) + from robonix_api.llm import metrics # noqa: PLC0415 + + class Response: + status_code = 200 + + @staticmethod + def json() -> dict: + return {"choices": [{"message": {"content": "summary"}}], + "usage": {"prompt_tokens": 7, "completion_tokens": 2}} + + def raise_for_status(self) -> None: + return None + + class Client: + def __init__(self, **_kwargs) -> None: + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc) -> None: + return None + + async def post(self, url, *, headers, json): + self.__class__.sent = (url, headers["Authorization"], json["model"]) + return Response() + + class Store: + @staticmethod + def query(filter_expr: str): + return [{"content": "the beacon is alpha-7"}] + + class Backend: + store = Store() + indexed: list = [] + + async def index_file(self, path) -> None: + self.indexed.append(path) + + httpx = types.ModuleType("httpx") + httpx.AsyncClient = Client + compact_module = types.ModuleType("memsearch.compact") + compact_module.COMPACT_PROMPT = "{chunks}" + with tempfile.TemporaryDirectory() as tmp: + module, _ = _load_service() + module.mem = Backend() + module.MEMORY_DIR = tmp + env = {"VLM_BASE_URL": "https://api.example.com/v1", "VLM_API_KEY": "k", + "VLM_MODEL": "m", metrics.DIRECTORY_ENV: tmp} + with patch.dict(os.environ, env), patch.dict( + sys.modules, {"httpx": httpx, "memsearch.compact": compact_module} + ): + out = asyncio.run(module.compact(_Empty())) + [record] = [json.loads(line) for line in + (Path(tmp) / f"memsearch-{os.getpid()}.jsonl").read_text().splitlines()] + summary = (Path(tmp) / "memory").glob("*.md") + self.assertIn("summary", next(summary).read_text()) + + self.assertIn("compacted successfully", out.data) + self.assertEqual(Client.sent, + ("https://api.example.com/v1/chat/completions", "Bearer k", "m")) + self.assertEqual(len(Backend.indexed), 1) + self.assertEqual((record["status"], record["input_tokens"], record["output_tokens"]), + ("ok", 7, 2)) + def test_start_hook_does_not_kill_database_holders(self) -> None: manifest = (PACKAGE_ROOT / "package_manifest.yaml").read_text(encoding="utf-8") self.assertNotIn("fuser", manifest) diff --git a/services/verifiers/vlm_verifier/vlm_verifier/vlm.py b/services/verifiers/vlm_verifier/vlm_verifier/vlm.py index 8dc45fc79..ac6e63c12 100644 --- a/services/verifiers/vlm_verifier/vlm_verifier/vlm.py +++ b/services/verifiers/vlm_verifier/vlm_verifier/vlm.py @@ -10,6 +10,7 @@ Requirements, api_key_for, catalog_from_env, + metrics, resolve_from_env, ) @@ -199,17 +200,20 @@ async def judge(config, envelope, before, after): ], } try: - async with httpx.AsyncClient(timeout=40, follow_redirects=False) as client: - headers = {"Content-Type": "application/json"} - if config.api_key: - headers["Authorization"] = "Bearer " + config.api_key - response = await client.post( - config.base_url + "/chat/completions", - headers=headers, - json=payload, - ) + with metrics.Call("vlm_verifier", "judge", config.base_url, config.model) as call: + async with httpx.AsyncClient(timeout=40, follow_redirects=False) as client: + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = "Bearer " + config.api_key + response = await client.post( + config.base_url + "/chat/completions", + headers=headers, + json=payload, + ) + call.finish(response) response.raise_for_status() - content = response.json()["choices"][0]["message"]["content"] + data = response.json() + content = data["choices"][0]["message"]["content"] except httpx.HTTPStatusError as exc: raise RuntimeError(f"VLM HTTP status {exc.response.status_code}") from None except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError): diff --git a/system/scene/scene_service/ingest/perception_vlm.py b/system/scene/scene_service/ingest/perception_vlm.py index 92e4a97e5..f67b4e619 100644 --- a/system/scene/scene_service/ingest/perception_vlm.py +++ b/system/scene/scene_service/ingest/perception_vlm.py @@ -23,6 +23,7 @@ Requirements, api_key_for, explicit_env_set, + metrics, resolve_from_env, ) @@ -341,8 +342,10 @@ async def _call_vlm(self, jpeg_b64: str) -> Optional[list[dict]]: } if self.reasoning_effort: body["reasoning_effort"] = self.reasoning_effort - async with httpx.AsyncClient(timeout=30.0) as client: - r = await client.post(url, json=body, headers=headers) + with metrics.Call("scene", "perception", self.base_url, self.model) as call: + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post(url, json=body, headers=headers) + call.finish(r) if r.status_code >= 400: log.warning("[scene-vlm] VLM HTTP %d: %s", r.status_code, r.text[:200]) return None diff --git a/system/scene/scene_service/object_captions.py b/system/scene/scene_service/object_captions.py index 57c0d6df8..1e9dbdcf1 100644 --- a/system/scene/scene_service/object_captions.py +++ b/system/scene/scene_service/object_captions.py @@ -65,7 +65,8 @@ async def caption_one(registry, views, map_binding, llm, tried: dict, _SYSTEM_PROMPT + _LANGUAGE.get( os.environ.get("SCENE_CAPTION_LANG", "zh"), _LANGUAGE["zh"]), f"The object is a {obj.label}.", - images=[base64.b64encode(jpeg).decode("ascii")]) + images=[base64.b64encode(jpeg).decode("ascii")], + operation="caption") caption = str((reply or {}).get("caption") or "").strip() if not caption: continue diff --git a/system/scene/scene_service/scene_graph/image_relations.py b/system/scene/scene_service/scene_graph/image_relations.py index 0560b7307..194bf3fa3 100644 --- a/system/scene/scene_service/scene_graph/image_relations.py +++ b/system/scene/scene_service/scene_graph/image_relations.py @@ -420,6 +420,7 @@ async def infer( user_message=user_message, timeout=_LLM_TIMEOUT_SEC, images=[image_b64], + operation="image_relations", ) except Exception as e: # noqa: BLE001 log.warning("[image-rel] VLM call failed: %s: %s", type(e).__name__, e) diff --git a/system/scene/scene_service/scene_graph/llm_client.py b/system/scene/scene_service/scene_graph/llm_client.py index b4ebf92d7..d7f7cadda 100644 --- a/system/scene/scene_service/scene_graph/llm_client.py +++ b/system/scene/scene_service/scene_graph/llm_client.py @@ -14,6 +14,7 @@ Requirements, api_key_for, explicit_env_set, + metrics, resolve_from_env, ) @@ -98,10 +99,12 @@ async def chat_json( *, timeout: float | None = None, images: list[str] | None = None, + operation: str = "chat", ) -> dict[str, Any]: """One request expecting a JSON object reply; {} on any failure. - `images` are base64 JPEGs sent as multimodal parts of the user turn.""" + `images` are base64 JPEGs sent as multimodal parts of the user turn. + `operation` names the request in the LLM metrics record.""" if not self.available: return {} @@ -130,10 +133,12 @@ async def chat_json( body["reasoning_effort"] = self.reasoning_effort try: - async with httpx.AsyncClient( - timeout=timeout or self.timeout - ) as client: - r = await client.post(url, json=body, headers=headers) + with metrics.Call("scene", operation, self.base_url, self.model) as call: + async with httpx.AsyncClient( + timeout=timeout or self.timeout + ) as client: + r = await client.post(url, json=body, headers=headers) + call.finish(r) if r.status_code >= 400: log.warning( "[scene-graph-llm] HTTP %d: %s", diff --git a/system/scene/scene_service/scene_graph/relations.py b/system/scene/scene_service/scene_graph/relations.py index 36c81d92a..7a15994df 100644 --- a/system/scene/scene_service/scene_graph/relations.py +++ b/system/scene/scene_service/scene_graph/relations.py @@ -151,6 +151,7 @@ async def _ask(self, source, target, system_prompt, user_msg, system_prompt=system_prompt, user_message=user_msg, timeout=_LLM_TIMEOUT_SEC, + operation="relations", ) if not raw: return SceneGraphEdge( diff --git a/system/scene/scripts/start.sh b/system/scene/scripts/start.sh index 4f4ab06b2..27150f9b2 100755 --- a/system/scene/scripts/start.sh +++ b/system/scene/scripts/start.sh @@ -267,6 +267,14 @@ if [[ -n "${SCRIBE_LOG_DIR:-}" && -d "${SCRIBE_LOG_DIR}" ]]; then EXTRA_MOUNTS+=(-v "${SCRIBE_LOG_DIR}:${SCRIBE_LOG_DIR}:ro") fi +# Model usage records: writable, at the same path, so the in-container writer +# appends to the deployment's llm-metrics directory. +declare -a METRICS_ARGS=() +if [[ -n "${ROBONIX_LLM_METRICS_DIR:-}" && -d "${ROBONIX_LLM_METRICS_DIR}" ]]; then + METRICS_ARGS+=(-v "${ROBONIX_LLM_METRICS_DIR}:${ROBONIX_LLM_METRICS_DIR}" + -e "ROBONIX_LLM_METRICS_DIR=${ROBONIX_LLM_METRICS_DIR}" -e "RBNX_BOOT_ID=${RBNX_BOOT_ID:-}") +fi + declare -a ZENOH_ARGS=() if [[ -n "${ROBONIX_ZENOH_ROUTER:-}" ]]; then ZENOH_ARGS=(-e "ROBONIX_ZENOH_ROUTER=${ROBONIX_ZENOH_ROUTER}") @@ -407,5 +415,6 @@ docker run --rm --cidfile "$RUN_DIR/cid" \ ${SOURCE_MOUNTS[@]+"${SOURCE_MOUNTS[@]}"} \ -v "$SCENE_HOST_DATA_DIR":/data/robonix \ ${EXTRA_MOUNTS[@]+"${EXTRA_MOUNTS[@]}"} \ + ${METRICS_ARGS[@]+"${METRICS_ARGS[@]}"} \ "$IMG" & wait "$!" diff --git a/uv.lock b/uv.lock index 2718dab70..572276257 100644 --- a/uv.lock +++ b/uv.lock @@ -3734,6 +3734,7 @@ dependencies = [ { name = "fastmcp" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "httpx" }, { name = "mcp" }, { name = "memsearch", extra = ["onnx"] }, { name = "numpy" }, @@ -3749,8 +3750,9 @@ requires-dist = [ { name = "fastmcp", specifier = ">=3,<5" }, { name = "grpcio", specifier = "==1.80.0" }, { name = "grpcio-tools", specifier = "==1.76.0" }, + { name = "httpx" }, { name = "mcp", specifier = ">=1.27,<2" }, - { name = "memsearch", extras = ["onnx"] }, + { name = "memsearch", extras = ["onnx"], specifier = ">=0.4.11,<0.5" }, { name = "numpy", specifier = ">=1.26,<2" }, { name = "onnxruntime", marker = "platform_machine != 'aarch64'", specifier = ">=1.23,<1.24" }, { name = "onnxruntime", marker = "platform_machine == 'aarch64'", specifier = "==1.18.1" }, From d14cc5f69a6f7b85fd27af796c27b61842b3e9f5 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 11:56:58 +0800 Subject: [PATCH 04/14] feat(rbnx): keep usage records under rbnx-boot/llm-metrics and add llm-usage --- Cargo.lock | 1 + tools/rbnx/Cargo.toml | 3 + tools/rbnx/README.md | 10 +- tools/rbnx/src/cmd/clean.rs | 46 +++++--- tools/rbnx/src/cmd/deploy.rs | 8 ++ tools/rbnx/src/cmd/llm_usage.rs | 201 ++++++++++++++++++++++++++++++++ tools/rbnx/src/cmd/mod.rs | 23 +++- 7 files changed, 275 insertions(+), 17 deletions(-) create mode 100644 tools/rbnx/src/cmd/llm_usage.rs diff --git a/Cargo.lock b/Cargo.lock index 6dad02068..9f718f8f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3229,6 +3229,7 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", + "tempfile", "tokio", "tokio-stream", "toml", diff --git a/tools/rbnx/Cargo.toml b/tools/rbnx/Cargo.toml index ed1d6f1f7..1273168f1 100644 --- a/tools/rbnx/Cargo.toml +++ b/tools/rbnx/Cargo.toml @@ -36,6 +36,9 @@ robonix-llm-catalog = { path = "../../system/llm-catalog" } robonix-pilot = { path = "../../system/pilot" } robonix-scribe.workspace = true +[dev-dependencies] +tempfile = "3" + [build-dependencies] robonix-codegen.workspace = true tonic-build.workspace = true diff --git a/tools/rbnx/README.md b/tools/rbnx/README.md index 074d5bc11..8dd0c8a31 100644 --- a/tools/rbnx/README.md +++ b/tools/rbnx/README.md @@ -51,6 +51,7 @@ cargo install --force --path tools/rbnx --bin rbnx | `rbnx init NAME [--path DIR]` | Create a robot deployment directory with a `robonix_manifest.yaml` | | `rbnx package-new NAME [-t primitive\|service\|skill] [--path DIR]` | Create a new package | | `rbnx logs [-d DIR] [-t TAG] [-l LEVEL] [-f] [--json] [--list-tags]` | Read Scribe log files, by default from `./rbnx-boot/logs` | +| `rbnx llm-usage [-d DIR] [--boot ID] [--json]` | Sum model usage records by consumer, operation, host and model, by default from `$ROBONIX_LLM_METRICS_DIR` or `./rbnx-boot/llm-metrics` | | `rbnx update [-y]` | Fast-forward cloned `url:` packages; asks first unless `-y`, and refuses without a terminal | | `rbnx update --check [--json]` | Report which `url:` packages have upstream commits; pulls nothing | | `rbnx version [--json]` | Version, git commit, build date, rustc, target; `--json` adds the source root | @@ -118,9 +119,12 @@ Next to the lock, `rbnx-boot/logs/` holds each component's log and `rbnx-boot/sessions/` holds session state: boot passes it to system components as `ROBONIX_SESSION_DIR` (an existing value wins), and Pilot keeps each session's transcript there so the session survives a restart. Boot clears -old log files but never touches `sessions/`. `rbnx clean -f` removes -everything under `rbnx-boot/` except `cache/` and `sessions/`; pass `--cache` -or `--sessions` to remove those too. +old log files but never touches `sessions/`. `rbnx-boot/llm-metrics/` holds +one JSON line per model request (see `robonix_api.llm`); boot exports it as +`ROBONIX_LLM_METRICS_DIR` (an existing value wins), never clears it, and +`rbnx llm-usage` sums it. `rbnx clean -f` removes everything under +`rbnx-boot/` except `cache/`, `sessions/` and `llm-metrics/`; pass +`--cache`, `--sessions` or `--llm-metrics` to remove each of those too. ## Built-in system components diff --git a/tools/rbnx/src/cmd/clean.rs b/tools/rbnx/src/cmd/clean.rs index 5eed1181f..e6a1afdc1 100644 --- a/tools/rbnx/src/cmd/clean.rs +++ b/tools/rbnx/src/cmd/clean.rs @@ -65,13 +65,19 @@ pub async fn execute( file: Option, cache: bool, sessions: bool, + llm_metrics: bool, ) -> Result<()> { + let wipe = Wipe { + cache, + sessions, + llm_metrics, + }; match (package, file) { (Some(_), Some(_)) => { anyhow::bail!("pass one of -p / -f, not both") } (Some(pkg), None) => clean_package(&pkg), - (None, Some(f)) => clean_deploy(&config, &f, cache, sessions), + (None, Some(f)) => clean_deploy(&config, &f, &wipe), (None, None) => { // Default-mode resolution. cwd-local hints in priority order: // 1. ./robonix_manifest.yaml → deploy clean (sibling of `rbnx boot -f`) @@ -80,7 +86,7 @@ pub async fn execute( let cwd = std::env::current_dir().context("get cwd")?; let deploy = cwd.join("robonix_manifest.yaml"); if deploy.is_file() { - return clean_deploy(&config, &deploy, cache, sessions); + return clean_deploy(&config, &deploy, &wipe); } match run_package::find_package_from_cwd() { Ok(pkg) => clean_package(&pkg), @@ -116,12 +122,17 @@ fn clean_package(pkg: &Path) -> Result<()> { Ok(()) } -fn clean_deploy( - config: &Config, - manifest_path: &Path, - also_cache: bool, - also_sessions: bool, -) -> Result<()> { +/// Which kept `rbnx-boot/` subdirectories `rbnx clean -f` also removes. +struct Wipe { + /// `--cache`: url-fetched package clones. + cache: bool, + /// `--sessions`: Pilot session transcripts. + sessions: bool, + /// `--llm-metrics`: per-request model usage records. + llm_metrics: bool, +} + +fn clean_deploy(config: &Config, manifest_path: &Path, wipe: &Wipe) -> Result<()> { let manifest_path = manifest_path .canonicalize() .with_context(|| format!("manifest not found: {}", manifest_path.display()))?; @@ -191,8 +202,10 @@ fn clean_deploy( // Deploy-level cleanup. Walk every entry under /rbnx-boot/ // and remove it. Skip cache/ unless `--cache` was given (re-cloning - // url-fetched packages is expensive), and sessions/ unless `--sessions` - // was given (it holds the transcripts that restore Pilot sessions). + // url-fetched packages is expensive), sessions/ unless `--sessions` was + // given (it holds the transcripts that restore Pilot sessions), and + // llm-metrics/ unless `--llm-metrics` was given (usage records a bill is + // computed from). let rbnx_boot = manifest_dir.join("rbnx-boot"); if rbnx_boot.exists() { if let Ok(entries) = std::fs::read_dir(&rbnx_boot) { @@ -200,14 +213,21 @@ fn clean_deploy( let path = entry.path(); let name = entry.file_name(); let is_cache = name == "cache"; - if is_cache && !also_cache { + if is_cache && !wipe.cache { output::sub_step(&format!("keep {} (use --cache to wipe)", path.display())); continue; } - if name == "sessions" && !also_sessions { + if name == "sessions" && !wipe.sessions { output::sub_step(&format!("keep {} (use --sessions to wipe)", path.display())); continue; } + if name == "llm-metrics" && !wipe.llm_metrics { + output::sub_step(&format!( + "keep {} (use --llm-metrics to wipe)", + path.display() + )); + continue; + } let label = if is_cache { " (cache)" } else { "" }; let kind = entry.file_type().ok(); if kind.map(|k| k.is_dir()).unwrap_or(false) { @@ -223,7 +243,7 @@ fn clean_deploy( } } } - // If rbnx-boot/ is empty (or only the kept cache/ and sessions/ remain), + // If rbnx-boot/ is empty (or only the kept subdirectories remain), // try to rmdir it too. Best-effort — no failure surface. let _ = std::fs::remove_dir(&rbnx_boot); } diff --git a/tools/rbnx/src/cmd/deploy.rs b/tools/rbnx/src/cmd/deploy.rs index 458d569f6..b6f9e207d 100644 --- a/tools/rbnx/src/cmd/deploy.rs +++ b/tools/rbnx/src/cmd/deploy.rs @@ -1829,6 +1829,13 @@ pub async fn execute( let session_dir = std::env::var_os("ROBONIX_SESSION_DIR") .map(PathBuf::from) .unwrap_or_else(|| manifest_dir.join("rbnx-boot").join("sessions")); + // Model usage records outlive the logs for the same reason; every child + // inherits the variable, so set it once here. An existing value wins. + let metrics_dir = std::env::var_os("ROBONIX_LLM_METRICS_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| manifest_dir.join("rbnx-boot").join("llm-metrics")); + std::fs::create_dir_all(&metrics_dir) + .with_context(|| format!("failed to create {}", metrics_dir.display()))?; // The CLI prepares and clears this directory before Scribe's first log // call. Do not remove files here: Scribe may already hold open handles. std::fs::create_dir_all(&log_dir) @@ -1841,6 +1848,7 @@ pub async fn execute( // Safety: called before any child spawns, no concurrent access. unsafe { std::env::set_var("SCRIBE_LOG_DIR", log_dir.as_os_str()); + std::env::set_var("ROBONIX_LLM_METRICS_DIR", metrics_dir.as_os_str()); } scribe::info( "bootstrap", diff --git a/tools/rbnx/src/cmd/llm_usage.rs b/tools/rbnx/src/cmd/llm_usage.rs new file mode 100644 index 000000000..d1b26b095 --- /dev/null +++ b/tools/rbnx/src/cmd/llm_usage.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: MulanPSL-2.0 +//! `rbnx llm-usage` — sum the per-request model usage records that +//! `robonix_api.llm.metrics` and `robonix-llm-metrics` append. + +use anyhow::{Context, Result}; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +/// Requests and reported tokens of one consumer/operation/host/model group. +#[derive(Debug, Default, PartialEq, Eq, Serialize)] +struct Total { + /// Component that made the requests. + consumer: String, + /// What the component used the model for. + operation: String, + /// Endpoint host and port; joined with `model` against a price list. + host: String, + /// Model named in the requests. + model: String, + /// Records in the group, one per call. + calls: u64, + /// Requests sent, retries included. + attempts: u64, + /// Calls per outcome (`ok`, `http_error`, ...). + status: BTreeMap, + /// Sum of reported prompt tokens. + input_tokens: u64, + /// Sum of reported generated tokens. + output_tokens: u64, + /// Sum of reported cached prompt tokens. + cached_input_tokens: u64, + /// Sum of reported reasoning tokens. + reasoning_tokens: u64, + /// Calls whose input or output count was not reported; their tokens are + /// missing from the sums. + unreported: u64, +} + +pub fn execute(dir: Option, boot: Option<&str>, json: bool) -> Result<()> { + let dir = dir.unwrap_or_else(|| { + std::env::var_os("ROBONIX_LLM_METRICS_DIR") + .filter(|d| !d.is_empty()) + .map_or_else(|| PathBuf::from("./rbnx-boot/llm-metrics"), PathBuf::from) + }); + let (totals, skipped) = sum(&dir, boot)?; + if json { + for total in &totals { + println!("{}", serde_json::to_string(total)?); + } + } else { + print_table(&totals); + } + if skipped > 0 { + eprintln!("rbnx llm-usage: skipped {skipped} unreadable line(s) or file(s)"); + } + Ok(()) +} + +/// Totals of every `*.jsonl` record under `dir`, and the count of lines +/// that were not a record: a line still being written, or an unreadable file +/// (counted as one). +fn sum(dir: &Path, boot: Option<&str>) -> Result<(Vec, u64)> { + let entries = std::fs::read_dir(dir).with_context(|| { + format!( + "{} is not readable; pass -d or set ROBONIX_LLM_METRICS_DIR", + dir.display() + ) + })?; + let mut groups: BTreeMap<[String; 4], Total> = BTreeMap::new(); + let mut skipped = 0; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("jsonl") { + continue; + } + let Ok(text) = std::fs::read_to_string(&path) else { + skipped += 1; + continue; + }; + for line in text.lines().filter(|line| !line.trim().is_empty()) { + let Ok(record) = serde_json::from_str::(line) else { + skipped += 1; + continue; + }; + if boot.is_some_and(|boot| record["boot_id"].as_str() != Some(boot)) { + continue; + } + let field = |name: &str| record[name].as_str().unwrap_or_default().to_owned(); + let key = [ + field("consumer"), + field("operation"), + field("host"), + field("model"), + ]; + let total = groups.entry(key.clone()).or_insert_with(|| { + let [consumer, operation, host, model] = key; + Total { + consumer, + operation, + host, + model, + ..Total::default() + } + }); + let count = |name: &str| record[name].as_u64(); + total.calls += 1; + total.attempts += count("attempts").unwrap_or(1); + *total.status.entry(field("status")).or_default() += 1; + total.input_tokens += count("input_tokens").unwrap_or(0); + total.output_tokens += count("output_tokens").unwrap_or(0); + total.cached_input_tokens += count("cached_input_tokens").unwrap_or(0); + total.reasoning_tokens += count("reasoning_tokens").unwrap_or(0); + if count("input_tokens").is_none() || count("output_tokens").is_none() { + total.unreported += 1; + } + } + } + Ok((groups.into_values().collect(), skipped)) +} + +fn print_table(totals: &[Total]) { + println!( + "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} status", + "consumer", + "operation", + "host", + "model", + "calls", + "attempts", + "input", + "output", + "cached", + "reasoning", + "unreported" + ); + for t in totals { + let status = t + .status + .iter() + .map(|(name, count)| format!("{name}={count}")) + .collect::>() + .join(" "); + println!( + "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} {status}", + t.consumer, + t.operation, + t.host, + t.model, + t.calls, + t.attempts, + t.input_tokens, + t.output_tokens, + t.cached_input_tokens, + t.reasoning_tokens, + t.unreported + ); + } +} + +#[cfg(test)] +mod tests { + use super::sum; + + #[test] + fn sums_by_group_and_boot_and_skips_partial_lines() { + let directory = tempfile::tempdir().unwrap(); + let records = [ + r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"ok","attempts":2,"input_tokens":10,"output_tokens":3,"cached_input_tokens":4,"boot_id":"a"}"#, + r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"timeout","attempts":1,"input_tokens":null,"output_tokens":null,"boot_id":"a"}"#, + r#"{"consumer":"scene","operation":"caption","host":"h","model":"m","status":"ok","input_tokens":5,"output_tokens":1,"boot_id":"b"}"#, + r#"{"consumer":"pilot","operation":"pl"#, + ]; + std::fs::write(directory.path().join("pilot-1.jsonl"), records.join("\n")).unwrap(); + std::fs::write(directory.path().join("notes.txt"), "not a record").unwrap(); + std::fs::write(directory.path().join("scene-2.jsonl"), [0xff, 0xfe]).unwrap(); + + let (totals, skipped) = sum(directory.path(), None).unwrap(); + assert_eq!(skipped, 2); + assert_eq!(totals.len(), 2); + let plan = &totals[0]; + assert_eq!((plan.calls, plan.attempts), (2, 3)); + assert_eq!( + ( + plan.input_tokens, + plan.output_tokens, + plan.cached_input_tokens + ), + (10, 3, 4) + ); + assert_eq!(plan.unreported, 1); + assert_eq!(plan.status.get("timeout"), Some(&1)); + // Python writers before `attempts` count as one request. + assert_eq!(totals[1].attempts, 1); + + let (totals, _) = sum(directory.path(), Some("b")).unwrap(); + assert_eq!(totals.len(), 1); + assert_eq!(totals[0].consumer, "scene"); + } +} diff --git a/tools/rbnx/src/cmd/mod.rs b/tools/rbnx/src/cmd/mod.rs index ad81828e1..cce26c844 100644 --- a/tools/rbnx/src/cmd/mod.rs +++ b/tools/rbnx/src/cmd/mod.rs @@ -24,6 +24,7 @@ mod init; mod inspect; mod install; mod list; +mod llm_usage; mod logs; mod package_new; mod path; @@ -188,6 +189,10 @@ pub enum Commands { /// transcripts, which restore sessions after a restart). #[arg(long)] sessions: bool, + /// With `-f`, also wipe `rbnx-boot/llm-metrics/` (per-request model + /// usage records, which `rbnx llm-usage` sums). + #[arg(long)] + llm_metrics: bool, }, /// Install a package from GitHub or local path /// @@ -434,6 +439,20 @@ pub enum Commands { #[arg(long)] list_tags: bool, }, + /// Sum the per-request model usage records by consumer, operation, host + /// and model: requests, outcomes and reported tokens. + LlmUsage { + /// Records directory (default: `$ROBONIX_LLM_METRICS_DIR`, else + /// `./rbnx-boot/llm-metrics`). + #[arg(short = 'd', long)] + dir: Option, + /// Only records of this boot (`RBNX_BOOT_ID`, `boot_id` in a record). + #[arg(long)] + boot: Option, + /// Print the totals as JSON lines instead of a table. + #[arg(long)] + json: bool, + }, } pub async fn execute(command: Commands, config: Config) -> Result<()> { @@ -488,7 +507,8 @@ pub async fn execute(command: Commands, config: Config) -> Result<()> { file, cache, sessions, - } => clean::execute(config, package, file, cache, sessions).await, + llm_metrics, + } => clean::execute(config, package, file, cache, sessions, llm_metrics).await, Commands::Install { github, path } => install::execute(config, github, path).await, Commands::List => list::execute(config).await, Commands::Info { name } => info::execute(config, &name).await, @@ -549,5 +569,6 @@ pub async fn execute(command: Commands, config: Config) -> Result<()> { json, list_tags, } => logs::execute(log_dir, tag, level, follow, json, list_tags).await, + Commands::LlmUsage { dir, boot, json } => llm_usage::execute(dir, boot.as_deref(), json), } } From c8999e21ebe64882129e448970a392bf75d605a2 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 04:14:50 +0000 Subject: [PATCH 05/14] fix(llm): record unreadable 2xx replies as transport errors --- pylib/robonix-api/robonix_api/llm/README.md | 2 +- pylib/robonix-api/robonix_api/llm/metrics.py | 17 ++++++++++++----- pylib/robonix-api/tests/test_llm_metrics.py | 10 +++++++++- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pylib/robonix-api/robonix_api/llm/README.md b/pylib/robonix-api/robonix_api/llm/README.md index add2fa931..5ac123aa1 100644 --- a/pylib/robonix-api/robonix_api/llm/README.md +++ b/pylib/robonix-api/robonix_api/llm/README.md @@ -181,7 +181,7 @@ one line per call and are never rotated: archive or remove them with | `host` | Endpoint host and port; never the path, query or credentials | | `model`, `provider_model` | Model requested, and the one the response names | | `stream` | Whether the reply was streamed | -| `status`, `http_status` | `ok` (2xx), `http_error`, `timeout`, `cancelled` or `transport_error` (also a streamed reply cut off before it finished) | +| `status`, `http_status` | `ok` (2xx with a JSON body), `http_error`, `timeout`, `cancelled` or `transport_error` (unreadable body, or a streamed reply cut off before it finished) | | `attempts` | Requests sent for this call, counting retries | | `input_tokens`, `output_tokens`, `cached_input_tokens`, `reasoning_tokens` | Reported counts; `null` when not reported | diff --git a/pylib/robonix-api/robonix_api/llm/metrics.py b/pylib/robonix-api/robonix_api/llm/metrics.py index d89d9aab6..91cdb4f13 100644 --- a/pylib/robonix-api/robonix_api/llm/metrics.py +++ b/pylib/robonix-api/robonix_api/llm/metrics.py @@ -104,17 +104,24 @@ def retry(self) -> None: def finish(self, response: Any = None) -> None: """Record an HTTP response (``status_code`` and ``json()``, as httpx has). - Without a response the call is ``ok`` with unknown usage. A body that is - not JSON leaves usage unknown; the request was still answered. + Without a response the call is ``ok`` with unknown usage. A 2xx reply + whose body is not JSON is ``transport_error``: the caller cannot read + the answer, even though the HTTP status looked successful. """ status = getattr(response, "status_code", None) + unreadable = False try: body = response.json() if response is not None else None except ValueError: body = None - self.record["status"] = ( - "ok" if status is None or 200 <= status < 300 else "http_error" - ) + unreadable = True + if status is None: + outcome = "ok" + elif 200 <= status < 300: + outcome = "transport_error" if unreadable else "ok" + else: + outcome = "http_error" + self.record["status"] = outcome self.record["http_status"] = status self.record.update(usage_from_response(body)) self._done = True diff --git a/pylib/robonix-api/tests/test_llm_metrics.py b/pylib/robonix-api/tests/test_llm_metrics.py index 6a9545910..18554037e 100644 --- a/pylib/robonix-api/tests/test_llm_metrics.py +++ b/pylib/robonix-api/tests/test_llm_metrics.py @@ -76,7 +76,8 @@ async def cancelled() -> None: statuses = [(r["host"], r["status"], r["http_status"]) for r in self.records()] self.assertEqual(statuses, [ - ("127.0.0.1:8000", "http_error", 429), ("", "ok", 200), + ("127.0.0.1:8000", "http_error", 429), + ("", "transport_error", 200), ("", "timeout", None), ("", "cancelled", None), ]) @@ -101,6 +102,13 @@ def test_only_a_2xx_reply_is_ok(self) -> None: ("http_error", 400), ("http_error", 500), ]) + def test_unreadable_2xx_body_is_a_transport_error(self) -> None: + with metrics.Call("scene", "decode", "https://h/v1", "m") as call: + call.finish(Response(200, "not json")) + [record] = self.records() + self.assertEqual((record["status"], record["http_status"]), ("transport_error", 200)) + self.assertIsNone(record["input_tokens"]) + def test_a_name_utf8_cannot_encode_still_records_the_usage(self) -> None: # A provider may answer with a lone surrogate; the record must survive it. body = json.loads('{"model": "\\ud800", "usage": {"prompt_tokens": 4, "completion_tokens": 1}}') From da5ac79034178f5b39f007a3a672a9f6837dd7f1 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 04:14:52 +0000 Subject: [PATCH 06/14] feat(rbnx): report latency in llm-usage --- tools/rbnx/README.md | 2 +- tools/rbnx/src/cmd/llm_usage.rs | 77 +++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/tools/rbnx/README.md b/tools/rbnx/README.md index 8dd0c8a31..faa6ac918 100644 --- a/tools/rbnx/README.md +++ b/tools/rbnx/README.md @@ -51,7 +51,7 @@ cargo install --force --path tools/rbnx --bin rbnx | `rbnx init NAME [--path DIR]` | Create a robot deployment directory with a `robonix_manifest.yaml` | | `rbnx package-new NAME [-t primitive\|service\|skill] [--path DIR]` | Create a new package | | `rbnx logs [-d DIR] [-t TAG] [-l LEVEL] [-f] [--json] [--list-tags]` | Read Scribe log files, by default from `./rbnx-boot/logs` | -| `rbnx llm-usage [-d DIR] [--boot ID] [--json]` | Sum model usage records by consumer, operation, host and model, by default from `$ROBONIX_LLM_METRICS_DIR` or `./rbnx-boot/llm-metrics` | +| `rbnx llm-usage [-d DIR] [--boot ID] [--json]` | Sum model usage records by consumer, operation, host and model—requests, outcomes, tokens and latency percentiles—by default from `$ROBONIX_LLM_METRICS_DIR` or `./rbnx-boot/llm-metrics` | | `rbnx update [-y]` | Fast-forward cloned `url:` packages; asks first unless `-y`, and refuses without a terminal | | `rbnx update --check [--json]` | Report which `url:` packages have upstream commits; pulls nothing | | `rbnx version [--json]` | Version, git commit, build date, rustc, target; `--json` adds the source root | diff --git a/tools/rbnx/src/cmd/llm_usage.rs b/tools/rbnx/src/cmd/llm_usage.rs index d1b26b095..45ad5a7fe 100644 --- a/tools/rbnx/src/cmd/llm_usage.rs +++ b/tools/rbnx/src/cmd/llm_usage.rs @@ -36,6 +36,37 @@ struct Total { /// Calls whose input or output count was not reported; their tokens are /// missing from the sums. unreported: u64, + /// Reported latencies, kept only to compute the summary below. + #[serde(skip)] + latencies: Vec, + /// Calls that reported a latency. + latency_samples: u64, + /// Median reported latency in milliseconds. + latency_p50_ms: Option, + /// 95th percentile reported latency in milliseconds. + latency_p95_ms: Option, + /// Maximum reported latency in milliseconds. + latency_max_ms: Option, +} + +impl Total { + /// Sort once, then fill the latency summary fields. + fn summarize_latency(&mut self) { + self.latency_samples = self.latencies.len() as u64; + if self.latencies.is_empty() { + return; + } + self.latencies.sort_unstable(); + self.latency_p50_ms = Some(percentile(&self.latencies, 0.50)); + self.latency_p95_ms = Some(percentile(&self.latencies, 0.95)); + self.latency_max_ms = self.latencies.last().copied(); + } +} + +/// Nearest-rank percentile of a sorted slice; `fraction` is in `0..=1`. +fn percentile(sorted: &[u64], fraction: f64) -> u64 { + let rank = (fraction * sorted.len() as f64).ceil() as usize; + sorted[rank.saturating_sub(1).min(sorted.len() - 1)] } pub fn execute(dir: Option, boot: Option<&str>, json: bool) -> Result<()> { @@ -112,17 +143,24 @@ fn sum(dir: &Path, boot: Option<&str>) -> Result<(Vec, u64)> { total.output_tokens += count("output_tokens").unwrap_or(0); total.cached_input_tokens += count("cached_input_tokens").unwrap_or(0); total.reasoning_tokens += count("reasoning_tokens").unwrap_or(0); + if let Some(latency) = count("latency_ms") { + total.latencies.push(latency); + } if count("input_tokens").is_none() || count("output_tokens").is_none() { total.unreported += 1; } } } - Ok((groups.into_values().collect(), skipped)) + let mut totals: Vec = groups.into_values().collect(); + for total in &mut totals { + total.summarize_latency(); + } + Ok((totals, skipped)) } fn print_table(totals: &[Total]) { println!( - "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} status", + "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} {:>7} {:>8} {:>8} {:>8} status", "consumer", "operation", "host", @@ -133,7 +171,11 @@ fn print_table(totals: &[Total]) { "output", "cached", "reasoning", - "unreported" + "unreported", + "lat_n", + "p50_ms", + "p95_ms", + "max_ms" ); for t in totals { let status = t @@ -143,7 +185,7 @@ fn print_table(totals: &[Total]) { .collect::>() .join(" "); println!( - "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} {status}", + "{:<14} {:<16} {:<24} {:<24} {:>6} {:>8} {:>10} {:>10} {:>10} {:>10} {:>10} {:>7} {:>8} {:>8} {:>8} {status}", t.consumer, t.operation, t.host, @@ -154,11 +196,20 @@ fn print_table(totals: &[Total]) { t.output_tokens, t.cached_input_tokens, t.reasoning_tokens, - t.unreported + t.unreported, + t.latency_samples, + fmt_ms(t.latency_p50_ms), + fmt_ms(t.latency_p95_ms), + fmt_ms(t.latency_max_ms) ); } } +/// A latency cell, or `-` when no record reported one. +fn fmt_ms(value: Option) -> String { + value.map_or_else(|| "-".to_owned(), |value| value.to_string()) +} + #[cfg(test)] mod tests { use super::sum; @@ -167,9 +218,9 @@ mod tests { fn sums_by_group_and_boot_and_skips_partial_lines() { let directory = tempfile::tempdir().unwrap(); let records = [ - r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"ok","attempts":2,"input_tokens":10,"output_tokens":3,"cached_input_tokens":4,"boot_id":"a"}"#, - r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"timeout","attempts":1,"input_tokens":null,"output_tokens":null,"boot_id":"a"}"#, - r#"{"consumer":"scene","operation":"caption","host":"h","model":"m","status":"ok","input_tokens":5,"output_tokens":1,"boot_id":"b"}"#, + r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"ok","attempts":2,"input_tokens":10,"output_tokens":3,"cached_input_tokens":4,"latency_ms":100,"boot_id":"a"}"#, + r#"{"consumer":"pilot","operation":"plan","host":"h","model":"m","status":"timeout","attempts":1,"input_tokens":null,"output_tokens":null,"latency_ms":300,"boot_id":"a"}"#, + r#"{"consumer":"scene","operation":"caption","host":"h","model":"m","status":"ok","input_tokens":5,"output_tokens":1,"latency_ms":200,"boot_id":"b"}"#, r#"{"consumer":"pilot","operation":"pl"#, ]; std::fs::write(directory.path().join("pilot-1.jsonl"), records.join("\n")).unwrap(); @@ -191,8 +242,18 @@ mod tests { ); assert_eq!(plan.unreported, 1); assert_eq!(plan.status.get("timeout"), Some(&1)); + assert_eq!( + ( + plan.latency_samples, + plan.latency_p50_ms, + plan.latency_p95_ms, + plan.latency_max_ms + ), + (2, Some(100), Some(300), Some(300)) + ); // Python writers before `attempts` count as one request. assert_eq!(totals[1].attempts, 1); + assert_eq!(totals[1].latency_p50_ms, Some(200)); let (totals, _) = sum(directory.path(), Some("b")).unwrap(); assert_eq!(totals.len(), 1); From 7a7e0ac92eca197f87abdee07e1d08edf773ccec Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 8 Nov 2026 08:53:01 +0000 Subject: [PATCH 07/14] fix(ci): run robonix-api tests with pytest --- .github/workflows/ci.yml | 4 +++- pylib/robonix-api/tests/test_shared_lifecycle.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 180cb7fce..154937ac4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,8 +93,10 @@ jobs: # tests rely on cannot pass by accident through PYTHONPATH alone. - name: Install robonix-api run: python3 -m pip install ./pylib/robonix-api + - name: Install test dependencies + run: python3 -m pip install pytest - name: Test robonix-api - run: python3 -m unittest discover -s pylib/robonix-api/tests -p 'test_*.py' -v + run: python3 -m pytest -q pylib/robonix-api/tests chronos-python: name: Chronos Python bindings diff --git a/pylib/robonix-api/tests/test_shared_lifecycle.py b/pylib/robonix-api/tests/test_shared_lifecycle.py index b89410acc..df9dea5dd 100644 --- a/pylib/robonix-api/tests/test_shared_lifecycle.py +++ b/pylib/robonix-api/tests/test_shared_lifecycle.py @@ -65,16 +65,18 @@ def _grpc_module(*, shared: bool, legacy: bool) -> SimpleNamespace: def test_all_legacy_driver_contracts_remain_available_for_old_manifests() -> None: - legacy_contracts = sorted( + driver_contracts = sorted( path for path in (ROOT / "capabilities").glob("**/driver.v1.toml") if path.parent.name != "lifecycle" ) + # The migration marked every contract it rewrote as Legacy. Later-added + # per-package contracts are not part of the old-manifest compatibility set. + legacy_contracts = [path for path in driver_contracts if "Legacy" in path.read_text()] assert len(legacy_contracts) == 13 for path in legacy_contracts: contents = path.read_text() - assert "Legacy" in contents, path assert "idl" in contents and "lifecycle/srv/Driver.srv" in contents, path From ff25ada7fbfe711cb24867b4c911d2512790b547 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 20:21:53 +0800 Subject: [PATCH 08/14] fix(pilot): keep usage metering in sync with the catalog branch --- system/pilot/src/vlm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/pilot/src/vlm.rs b/system/pilot/src/vlm.rs index 68fb50575..637c401ad 100644 --- a/system/pilot/src/vlm.rs +++ b/system/pilot/src/vlm.rs @@ -778,7 +778,6 @@ impl VlmClient { return Err(error).context("open VLM chat stream"); } }; - }; let status = response.status(); if status.is_success() { // Until the stream ends, dropping it means the caller gave up. @@ -1479,6 +1478,7 @@ mod tests { api_base: base.to_string(), api_key: "local-test".into(), model: "local-test".into(), + reasoning_effort: None, configured_context_window_tokens: None, prompt_cache_key: "local-test".into(), metrics_dir: Some(metrics_dir.to_path_buf()), From 314afd36e5eda48a4d1389c2e08db5fc487f0761 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:11:50 +0800 Subject: [PATCH 09/14] fix(ci): leave the atlas binding test to its own job --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 154937ac4..e0a45baa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,11 @@ jobs: - name: Install test dependencies run: python3 -m pip install pytest - name: Test robonix-api - run: python3 -m pytest -q pylib/robonix-api/tests + # test_atlas_time_descriptor needs generated atlas bindings; the + # atlas-python job builds and covers it. + run: >- + python3 -m pytest -q pylib/robonix-api/tests + --ignore=pylib/robonix-api/tests/test_atlas_time_descriptor.py chronos-python: name: Chronos Python bindings From bb14bc51f08cab3715564ba8832b6529df39a356 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:18:41 +0800 Subject: [PATCH 10/14] fix(tests): skip the atlas metadata test without generated bindings --- .github/workflows/ci.yml | 6 +----- .../tests/test_atlas_time_descriptor.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0a45baa0..154937ac4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,11 +96,7 @@ jobs: - name: Install test dependencies run: python3 -m pip install pytest - name: Test robonix-api - # test_atlas_time_descriptor needs generated atlas bindings; the - # atlas-python job builds and covers it. - run: >- - python3 -m pytest -q pylib/robonix-api/tests - --ignore=pylib/robonix-api/tests/test_atlas_time_descriptor.py + run: python3 -m pytest -q pylib/robonix-api/tests chronos-python: name: Chronos Python bindings diff --git a/pylib/robonix-api/tests/test_atlas_time_descriptor.py b/pylib/robonix-api/tests/test_atlas_time_descriptor.py index d4f05b82f..c96c37c99 100644 --- a/pylib/robonix-api/tests/test_atlas_time_descriptor.py +++ b/pylib/robonix-api/tests/test_atlas_time_descriptor.py @@ -9,9 +9,6 @@ from types import SimpleNamespace from unittest.mock import Mock -import atlas_pb2 -import atlas_pb2_grpc -import grpc import pytest from robonix_api import CapabilityTimeDescriptor, Primitive, Ros2Params, Transport @@ -20,6 +17,18 @@ ROOT = Path(__file__).resolve().parents[3] +# These bindings and the atlas binary are generated by the atlas-python job; a +# plain robonix-api run has neither and should skip rather than fail collection. +try: + import atlas_pb2 + import atlas_pb2_grpc + import grpc +except ImportError as exc: + pytest.skip(f"atlas bindings are not generated: {exc}", allow_module_level=True) + +if not (Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) / "debug/robonix-atlas").exists(): + pytest.skip("robonix-atlas binary is not built", allow_module_level=True) + @pytest.fixture def atlas(tmp_path, monkeypatch): From 41187338295a2b87206b53715246f7fd352c9d09 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:35:03 +0800 Subject: [PATCH 11/14] fix(llm): forward the usage-records directory through the catalog launcher --- pylib/robonix-api/robonix_api/llm/runtime_env.txt | 1 + pylib/robonix-api/tests/test_llm_launcher.py | 2 ++ tools/rbnx/src/cmd/deploy.rs | 3 +++ 3 files changed, 6 insertions(+) diff --git a/pylib/robonix-api/robonix_api/llm/runtime_env.txt b/pylib/robonix-api/robonix_api/llm/runtime_env.txt index 4350ae109..d34b4ccf8 100644 --- a/pylib/robonix-api/robonix_api/llm/runtime_env.txt +++ b/pylib/robonix-api/robonix_api/llm/runtime_env.txt @@ -32,6 +32,7 @@ ROBONIX_DRIVER_CONTRACT_ID ROBONIX_DRIVER_ALLOW_OLD_ARTIFACT_FALLBACK ROBONIX_SOURCE_PATH ROBONIX_SESSION_DIR ROBONIX_SOMA_ENDPOINT ROBONIX_FORCE_CPU ROBONIX_ZENOH_ROUTER ROBONIX_ZENOH_MODE ROBONIX_ZENOH_LISTEN ROBONIX_CONFIG_PATH ROBONIX_LLM_CATALOG ROBONIX_LLM_KEY_ENVS ROBONIX_LLM_FEATURES +ROBONIX_LLM_METRICS_DIR ROBONIX_LLM_INPUT_TOKENS ROBONIX_LLM_MAX_OUTPUT_TOKENS ROBONIX_PILOT_LISTEN ROBONIX_PILOT_PROVIDER_ID ROBONIX_PILOT_SOUL ROBONIX_PILOT_MAX_TOOL_ROUNDS ROBONIX_PILOT_VLM_IDLE_TIMEOUT_SECS diff --git a/pylib/robonix-api/tests/test_llm_launcher.py b/pylib/robonix-api/tests/test_llm_launcher.py index 184266d29..d4a0d1c77 100644 --- a/pylib/robonix-api/tests/test_llm_launcher.py +++ b/pylib/robonix-api/tests/test_llm_launcher.py @@ -25,6 +25,7 @@ def _run(self, catalog, **extra_env): "UNRELATED_CREDENTIAL": "unused", "ROBONIX_LLM_KEY_ENVS": "UNRELATED_CREDENTIAL", "RBNX_BOOT_ID": "test-boot", + "ROBONIX_LLM_METRICS_DIR": "/tmp/llm-metrics", "RBNX_INSTANCE_NAME": "probe", "ROS_DOMAIN_ID": "31", "SCENE_DUALMAP_SAM_WEIGHTS": "/models/sam.pt", @@ -52,6 +53,7 @@ def test_catalog_keys_and_runtime_settings_reach_the_consumer(self): self.assertEqual(env["MODEL_AUTH"], "catalog-value") self.assertEqual(env["ROBONIX_LLM_KEY_ENVS"], "MODEL_AUTH") self.assertEqual(env["RBNX_BOOT_ID"], "test-boot") + self.assertEqual(env["ROBONIX_LLM_METRICS_DIR"], "/tmp/llm-metrics") self.assertEqual(env["RBNX_INSTANCE_NAME"], "probe") self.assertEqual(env["ROS_DOMAIN_ID"], "31") self.assertEqual(env["SCENE_DUALMAP_SAM_WEIGHTS"], "/models/sam.pt") diff --git a/tools/rbnx/src/cmd/deploy.rs b/tools/rbnx/src/cmd/deploy.rs index b6f9e207d..048b895b7 100644 --- a/tools/rbnx/src/cmd/deploy.rs +++ b/tools/rbnx/src/cmd/deploy.rs @@ -544,6 +544,7 @@ soma_endpoint: 127.0.0.1:50091 "MEMGRAPH_LLM_BASE_URL", "MEMGRAPH_LLM_API_KEY", "ROBONIX_CONFIG_PATH", + "ROBONIX_LLM_METRICS_DIR", "RBNX_BOOT_ID", "UNUSED_PROVIDER_API_KEY", "UNRELATED_CREDENTIAL", @@ -603,6 +604,7 @@ soma_endpoint: 127.0.0.1:50091 std::env::set_var("UNUSED_PROVIDER_API_KEY", "unused"); std::env::set_var("UNRELATED_CREDENTIAL", "unused"); std::env::set_var("RBNX_BOOT_ID", "test-boot"); + std::env::set_var("ROBONIX_LLM_METRICS_DIR", "/tmp/llm-metrics"); } for (explicit_key, config, expected_key) in [ (None, "{}", ""), @@ -623,6 +625,7 @@ soma_endpoint: 127.0.0.1:50091 cmd.arg("-c").arg(concat!( "test \"$CAT_KEY\" = secret && ", "test \"$RBNX_BOOT_ID\" = test-boot && ", + "test \"$ROBONIX_LLM_METRICS_DIR\" = /tmp/llm-metrics && ", "test \"$ROBONIX_LLM_KEY_ENVS\" = CAT_KEY && ", "test -n \"$ROBONIX_LLM_CATALOG\" && test -n \"$PATH\" && ", "test \"${UNUSED_PROVIDER_API_KEY+x}\" != x && ", From 7ed2cbcec808dd619837164ce10016f933070668 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:35:03 +0800 Subject: [PATCH 12/14] fix(llm): fix the merged docs and keyless compact requests --- pylib/robonix-api/README.md | 5 +++-- pylib/robonix-api/robonix_api/llm/README.md | 12 +++++++----- .../robonix-api/tests/test_atlas_time_descriptor.py | 4 +++- services/memsearch/memsearch_service/service.py | 6 +++++- system/pilot/src/vlm.rs | 5 ++--- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pylib/robonix-api/README.md b/pylib/robonix-api/README.md index 6bd988f8d..ccdef0366 100644 --- a/pylib/robonix-api/README.md +++ b/pylib/robonix-api/README.md @@ -106,8 +106,9 @@ The Atlas wire protocol (atlas_pb2 / atlas_pb2_grpc) is pre-generated and bundle `robonix_api.llm` is the shared model catalog and resolver used by Scene, Memory, Memsearch, the VLM Verifier, CI scripts and other consumers. It answers "which model endpoint should this request use?" from a declarative -catalog and leaves the provider call to the consumer's own client. It does not -proxy traffic, hold keys or track health. See +catalog and leaves the provider call to the consumer's own client. The +`metrics` helper appends one JSON line per model request for usage accounting. +It does not proxy traffic, hold keys or track health. See [`robonix_api/llm/README.md`](robonix_api/llm/README.md) for the catalog format, the Python API and the deployment variables. diff --git a/pylib/robonix-api/robonix_api/llm/README.md b/pylib/robonix-api/robonix_api/llm/README.md index 5ac123aa1..c0f17e380 100644 --- a/pylib/robonix-api/robonix_api/llm/README.md +++ b/pylib/robonix-api/robonix_api/llm/README.md @@ -1,10 +1,11 @@ # robonix_api.llm -Model catalog and model resolver for Robonix consumers, shipped as the -`robonix_api.llm` submodule of `robonix-api`. +Model catalog, model resolver and per-request usage records for Robonix +consumers, shipped as the `robonix_api.llm` submodule of `robonix-api`. Consumers resolve an endpoint from their requirements and connect directly; -the catalog adds no proxy or service. +the catalog adds no proxy or service. `metrics` records one JSON line per +model call so usage can be counted across consumers. ## ModelCatalog @@ -135,6 +136,7 @@ The requirements used by the in-tree consumers: The Rust implementation lives in `system/llm-catalog` and reads the same catalog with the same rules. The shared cases in `robonix_api/llm/contract/resolve_cases.json` are run by both test suites. + ## Model usage records Shared helpers for Robonix components that call a model. `metrics` records @@ -208,7 +210,6 @@ with the cases in [contract/usage_cases.json](contract/usage_cases.json). ```bash PYTHONPATH=pylib/robonix-api python3 -m unittest discover \ -s pylib/robonix-api/tests -p 'test_llm_*.py' -v -PYTHONPATH=pylib/robonix-api python3 -m unittest pylib/robonix-api/tests/test_llm_metrics.py cargo test -p robonix-llm-catalog cargo test -p robonix-llm-metrics ``` @@ -220,4 +221,5 @@ calls it directly with a generated image: ```bash export DEEPSEEK_API_KEY=... -PYTHONPATH=pylib/robonix-api python3 pylib/robonix-api/examples/llm_deepseek_smoke.py``` +PYTHONPATH=pylib/robonix-api python3 pylib/robonix-api/examples/llm_deepseek_smoke.py +``` diff --git a/pylib/robonix-api/tests/test_atlas_time_descriptor.py b/pylib/robonix-api/tests/test_atlas_time_descriptor.py index c96c37c99..e3d6123b1 100644 --- a/pylib/robonix-api/tests/test_atlas_time_descriptor.py +++ b/pylib/robonix-api/tests/test_atlas_time_descriptor.py @@ -27,7 +27,9 @@ pytest.skip(f"atlas bindings are not generated: {exc}", allow_module_level=True) if not (Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")) / "debug/robonix-atlas").exists(): - pytest.skip("robonix-atlas binary is not built", allow_module_level=True) + # Bindings but no binary means the job that generated them also owes a + # build; a silent skip there would hide a real regression. + pytest.fail("atlas bindings exist but target/debug/robonix-atlas is missing", pytrace=False) @pytest.fixture diff --git a/services/memsearch/memsearch_service/service.py b/services/memsearch/memsearch_service/service.py index 7f3e32936..e93c98198 100644 --- a/services/memsearch/memsearch_service/service.py +++ b/services/memsearch/memsearch_service/service.py @@ -281,11 +281,15 @@ async def _compact(base_url: str, api_key: str, model: str) -> Path | None: if not chunks: return None prompt = COMPACT_PROMPT.format(chunks="\n\n---\n\n".join(c["content"] for c in chunks)) + headers = {"Content-Type": "application/json"} + if api_key: + # Keyless local endpoints reject a bare "Bearer ". + headers["Authorization"] = f"Bearer {api_key}" with metrics.Call("memsearch", "compact", base_url, model) as call: async with httpx.AsyncClient(timeout=120.0) as client: response = await client.post( f"{base_url.rstrip('/')}/chat/completions", - headers={"Authorization": f"Bearer {api_key}"}, + headers=headers, json={"model": model, "messages": [{"role": "user", "content": prompt}]}, ) call.finish(response) diff --git a/system/pilot/src/vlm.rs b/system/pilot/src/vlm.rs index 637c401ad..41c55037f 100644 --- a/system/pilot/src/vlm.rs +++ b/system/pilot/src/vlm.rs @@ -897,8 +897,8 @@ impl VlmClient { // Walk the upstream chunk-by-chunk, accumulating tool-call deltas by // index until the upstream finishes; then emit one ToolCall per index - // and a final Finish event. Use mpsc + spawn so we can return the - // boxed Stream while the polling runs in the background. + // and a final Finish event. Use mpsc + spawn so the caller keeps + // reading while this task polls the upstream. let (tx, rx) = tokio::sync::mpsc::channel::>(64); tokio::spawn(async move { let mut tc_acc: BTreeMap = BTreeMap::new(); @@ -1004,7 +1004,6 @@ impl VlmClient { return; } } - let _ = finish; // surface to PilotEvent later if needed let _ = tx.send(Ok(VlmStreamItem::Finish)).await; }); From 6ef89f3a7b2ffc85c0900eebf717483be874ee51 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:43:34 +0800 Subject: [PATCH 13/14] fix(llm): finish the usage-metering review follow-ups --- pylib/robonix-api/robonix_api/llm/README.md | 6 ++++-- pylib/robonix-api/robonix_api/llm/metrics.py | 8 +++++--- system/scene/scene_service/scene_graph/llm_client.py | 2 +- system/scene/scripts/start.sh | 4 +++- system/scene/tests/test_image_relations.py | 4 ++-- system/scene/tests/test_scene_graph.py | 6 +++--- tools/rbnx/src/cmd/llm_usage.rs | 2 +- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/pylib/robonix-api/robonix_api/llm/README.md b/pylib/robonix-api/robonix_api/llm/README.md index c0f17e380..0d54ab1a7 100644 --- a/pylib/robonix-api/robonix_api/llm/README.md +++ b/pylib/robonix-api/robonix_api/llm/README.md @@ -171,8 +171,10 @@ must match `[a-z0-9_-]+`, because it names the file, and `Call` raises `rbnx clean -f` keep unless `--llm-metrics` is given. Each record is appended with one write, so a reader sees whole lines except possibly the one being written; `rbnx llm-usage` skips such a line and sums the rest. The files grow by -one line per call and are never rotated: archive or remove them with -`rbnx clean -f --llm-metrics` once their totals are taken. +one line per call and are never rotated. Each record is a few hundred bytes, +so the directory grows with call volume; `rbnx llm-usage` reads every record it +sums, so archive or remove the directory with `rbnx clean -f --llm-metrics` +when it reaches a size you are willing to parse in one run. | Field | Meaning | |---|---| diff --git a/pylib/robonix-api/robonix_api/llm/metrics.py b/pylib/robonix-api/robonix_api/llm/metrics.py index 91cdb4f13..5a68503b1 100644 --- a/pylib/robonix-api/robonix_api/llm/metrics.py +++ b/pylib/robonix-api/robonix_api/llm/metrics.py @@ -147,9 +147,11 @@ def write(record: dict[str, Any]) -> None: directory = os.environ.get(DIRECTORY_ENV, "").strip() if not _announced: # Without this line an unset variable loses every record silently. - _announced = True - log.info("llm metrics: %s", f"records -> {directory}" if directory - else f"off ({DIRECTORY_ENV} unset)") + with _lock: + first, _announced = not _announced, True + if first: + log.info("llm metrics: %s", f"records -> {directory}" if directory + else f"off ({DIRECTORY_ENV} unset)") if not directory: return line = dict(record, ts_ms=int(time.time() * 1000), diff --git a/system/scene/scene_service/scene_graph/llm_client.py b/system/scene/scene_service/scene_graph/llm_client.py index d7f7cadda..d909b6d6c 100644 --- a/system/scene/scene_service/scene_graph/llm_client.py +++ b/system/scene/scene_service/scene_graph/llm_client.py @@ -99,7 +99,7 @@ async def chat_json( *, timeout: float | None = None, images: list[str] | None = None, - operation: str = "chat", + operation: str, ) -> dict[str, Any]: """One request expecting a JSON object reply; {} on any failure. diff --git a/system/scene/scripts/start.sh b/system/scene/scripts/start.sh index 27150f9b2..1ffe2eb76 100755 --- a/system/scene/scripts/start.sh +++ b/system/scene/scripts/start.sh @@ -270,7 +270,9 @@ fi # Model usage records: writable, at the same path, so the in-container writer # appends to the deployment's llm-metrics directory. declare -a METRICS_ARGS=() -if [[ -n "${ROBONIX_LLM_METRICS_DIR:-}" && -d "${ROBONIX_LLM_METRICS_DIR}" ]]; then +if [[ -n "${ROBONIX_LLM_METRICS_DIR:-}" ]]; then + # Standalone launches may name a directory that does not exist yet. + mkdir -p "${ROBONIX_LLM_METRICS_DIR}" METRICS_ARGS+=(-v "${ROBONIX_LLM_METRICS_DIR}:${ROBONIX_LLM_METRICS_DIR}" -e "ROBONIX_LLM_METRICS_DIR=${ROBONIX_LLM_METRICS_DIR}" -e "RBNX_BOOT_ID=${RBNX_BOOT_ID:-}") fi diff --git a/system/scene/tests/test_image_relations.py b/system/scene/tests/test_image_relations.py index d0603d576..3df2280a8 100644 --- a/system/scene/tests/test_image_relations.py +++ b/system/scene/tests/test_image_relations.py @@ -143,7 +143,7 @@ def patched(*a, **k): try: c = SceneGraphLLMClient(api_key="k", base_url="http://x", reasoning_effort="") # With an image: content is a list [text, image_url]. - run(c.chat_json("sys", "look", images=["QUJD"])) + run(c.chat_json("sys", "look", images=["QUJD"], operation="test")) content = captured["body"]["messages"][1]["content"] assert isinstance(content, list) assert content[0]["type"] == "text" and content[0]["text"] == "look" @@ -151,7 +151,7 @@ def patched(*a, **k): assert content[1]["image_url"]["url"] == "data:image/jpeg;base64,QUJD" # Without images: content is the bare string (backward compatible). captured.clear() - run(c.chat_json("sys", "plain")) + run(c.chat_json("sys", "plain", operation="test")) assert captured["body"]["messages"][1]["content"] == "plain" finally: httpx.AsyncClient = orig diff --git a/system/scene/tests/test_scene_graph.py b/system/scene/tests/test_scene_graph.py index 7313605e2..1e54e0fc6 100644 --- a/system/scene/tests/test_scene_graph.py +++ b/system/scene/tests/test_scene_graph.py @@ -238,7 +238,7 @@ def test_llm_client_no_key(): assert not client.available result = _ensure_loop().run_until_complete( - client.chat_json("system", "user") + client.chat_json("system", "user", operation="test") ) assert result == {} finally: @@ -678,13 +678,13 @@ def patched(*a, **k): c = SceneGraphLLMClient( api_key="k", base_url="http://x", reasoning_effort="minimal" ) - run(c.chat_json("sys", "user")) + run(c.chat_json("sys", "user", operation="test")) assert captured["body"]["reasoning_effort"] == "minimal" assert "max_tokens" not in captured["body"] captured.clear() c2 = SceneGraphLLMClient(api_key="k", base_url="http://x", reasoning_effort="") - run(c2.chat_json("sys", "user")) + run(c2.chat_json("sys", "user", operation="test")) assert "reasoning_effort" not in captured["body"] finally: httpx.AsyncClient = orig diff --git a/tools/rbnx/src/cmd/llm_usage.rs b/tools/rbnx/src/cmd/llm_usage.rs index 45ad5a7fe..d0f848b5d 100644 --- a/tools/rbnx/src/cmd/llm_usage.rs +++ b/tools/rbnx/src/cmd/llm_usage.rs @@ -251,7 +251,7 @@ mod tests { ), (2, Some(100), Some(300), Some(300)) ); - // Python writers before `attempts` count as one request. + // A record without `attempts` (older writer or damaged line) is one request. assert_eq!(totals[1].attempts, 1); assert_eq!(totals[1].latency_p50_ms, Some(200)); From 3cdd09aed279fc6fe37ad09fb15bf47399ab85e5 Mon Sep 17 00:00:00 2001 From: Anlai Lu Date: Sun, 15 Nov 2026 21:55:42 +0800 Subject: [PATCH 14/14] fix(llm): tighten the usage docs and cover keyless compact --- AGENTS.md | 5 +- Cargo.toml | 7 ++- pylib/robonix-api/robonix_api/llm/README.md | 9 ++-- services/memsearch/tests/test_lifecycle.py | 56 +++++++++++++++++++++ tools/rbnx/src/cmd/llm_usage.rs | 2 +- 5 files changed, 70 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e09e8988a..6426d24bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,8 +41,9 @@ Third-party skills are not vendored into the repository. These are recommended f Robonix is an **embodied AI operating system**: a monorepo with Rust system components, Python packages, capability contracts, IDL libraries, examples, and documentation. The Rust workspace lives at the repository root in `Cargo.toml`; -current Rust members are `system/{atlas,executor,liaison,llm-metrics,pilot,scribe,soma,vitals}` -and `tools/{rbnx,codegen}`. The thirteen system component directories live under +current Rust members are +`system/{atlas,chronos,executor,liaison,llm-catalog,llm-metrics,pilot,scribe,soma,vitals}` +and `tools/{rbnx,codegen}`. The system component directories live under `system/`; several are documentation/planning components today, while `system/scene` is a Python package managed by uv. Python workspace packages are listed in the root `pyproject.toml`: shared libraries in `pylib/*`, reference diff --git a/Cargo.toml b/Cargo.toml index 75063a284..3abf99303 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,12 @@ [workspace] -# The 12 Robonix system components live one-per-directory under `system/`. -# Seven currently have a Rust implementation: +# Robonix system components live one-per-directory under `system/`. +# These currently have a Rust implementation: # - atlas (capability discovery) +# - chronos (device clock library) # - executor (runs RTDL programs, dispatching each call to its provider) # - liaison (human-machine interaction) +# - llm-catalog (model catalog and resolver) +# - llm-metrics (per-request model usage records) # - pilot (turns a task into an RTDL program and validates it) # - scribe (structured logging library) # - soma (body description; launches primitive and skill packages) diff --git a/pylib/robonix-api/robonix_api/llm/README.md b/pylib/robonix-api/robonix_api/llm/README.md index 0d54ab1a7..354f8e216 100644 --- a/pylib/robonix-api/robonix_api/llm/README.md +++ b/pylib/robonix-api/robonix_api/llm/README.md @@ -115,7 +115,6 @@ Containerized Scene mounts the catalog at the same absolute path and forwards on supplies that list; standalone launches derive it with the host's Python and `robonix_api.llm` (PyYAML is needed for YAML catalogs). Without a catalog, Scene forwards only the legacy `VLM_API_KEY` and `OPENAI_API_KEY` variables. -The catalog is read once per process; restart a consumer after editing it. The requirements used by the in-tree consumers: @@ -191,9 +190,11 @@ when it reaches a size you are willing to parse in one run. Prompts, replies and keys are never recorded. Prices are not recorded either: cost is token counts times the operator's price list, joined on `host` + -`model` against the totals `rbnx llm-usage --json` prints. A launcher that -filters the environment must pass `ROBONIX_LLM_METRICS_DIR` and `RBNX_BOOT_ID` -through; otherwise its consumers write nothing, which their startup log says. +`model` against the totals `rbnx llm-usage --json` prints. Whether a consumer +writes records depends on `ROBONIX_LLM_METRICS_DIR`: a launcher that filters the +environment must pass it through, and the startup log says when it is off. +`RBNX_BOOT_ID` only attributes a record to a boot; without it records still +appear with `boot_id: null`. The Rust crate `system/llm-metrics` writes the same record. Both parse usage with the cases in [contract/usage_cases.json](contract/usage_cases.json). diff --git a/services/memsearch/tests/test_lifecycle.py b/services/memsearch/tests/test_lifecycle.py index e37eaef54..e92c71c7e 100644 --- a/services/memsearch/tests/test_lifecycle.py +++ b/services/memsearch/tests/test_lifecycle.py @@ -349,6 +349,62 @@ async def index_file(self, path) -> None: self.assertEqual((record["status"], record["input_tokens"], record["output_tokens"]), ("ok", 7, 2)) + def test_keyless_compact_omits_the_authorization_header(self) -> None: + import asyncio + + from robonix_api.llm import metrics # noqa: PLC0415 + + class Response: + status_code = 200 + + @staticmethod + def json() -> dict: + return {"choices": [{"message": {"content": "summary"}}]} + + def raise_for_status(self) -> None: + return None + + class Client: + headers: dict | None = None + + def __init__(self, **_kwargs) -> None: + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc) -> None: + return None + + async def post(self, url, *, headers, json): + Client.headers = headers + return Response() + + class Store: + @staticmethod + def query(filter_expr: str): + return [{"content": "the beacon is alpha-7"}] + + class Backend: + store = Store() + + async def index_file(self, path) -> None: + return None + + httpx = types.ModuleType("httpx") + httpx.AsyncClient = Client + compact_module = types.ModuleType("memsearch.compact") + compact_module.COMPACT_PROMPT = "{chunks}" + with tempfile.TemporaryDirectory() as tmp: + module, _ = _load_service() + module.mem = Backend() + module.MEMORY_DIR = tmp + with patch.dict(os.environ, {metrics.DIRECTORY_ENV: tmp}), patch.dict( + sys.modules, {"httpx": httpx, "memsearch.compact": compact_module} + ): + asyncio.run(module._compact("https://local.example/v1", "", "m")) + self.assertEqual(Client.headers, {"Content-Type": "application/json"}) + def test_start_hook_does_not_kill_database_holders(self) -> None: manifest = (PACKAGE_ROOT / "package_manifest.yaml").read_text(encoding="utf-8") self.assertNotIn("fuser", manifest) diff --git a/tools/rbnx/src/cmd/llm_usage.rs b/tools/rbnx/src/cmd/llm_usage.rs index d0f848b5d..8741d4cce 100644 --- a/tools/rbnx/src/cmd/llm_usage.rs +++ b/tools/rbnx/src/cmd/llm_usage.rs @@ -251,7 +251,7 @@ mod tests { ), (2, Some(100), Some(300), Some(300)) ); - // A record without `attempts` (older writer or damaged line) is one request. + // A record without `attempts` (an older writer) is one request. assert_eq!(totals[1].attempts, 1); assert_eq!(totals[1].latency_p50_ms, Some(200));