From 9fb95f9bb24888c5f80ad08ce4979463c336c9ab Mon Sep 17 00:00:00 2001 From: Robert Kim Date: Mon, 27 Apr 2026 16:34:59 +0000 Subject: [PATCH 1/9] feat: add hermes-plugin example (Laminar tracing for Hermes Agent) (LAM-1511) Ships as examples/hermes-plugin: a standalone pip package that bridges nousresearch/hermes-agent plugin hooks (pre_llm_call, pre_tool_call, post_tool_call, post_api_request, post_llm_call, on_session_end, subagent_stop) to Laminar spans. Entry-point install exposes it to Hermes's PluginManager without needing to publish to PyPI. Co-Authored-By: Claude Opus 4.7 --- CLAUDE.md | 7 + examples/hermes-plugin/README.md | 52 +++ examples/hermes-plugin/pyproject.toml | 26 ++ .../hermes-plugin/src/lmnr_hermes/__init__.py | 411 ++++++++++++++++++ .../hermes-plugin/src/lmnr_hermes/plugin.yaml | 17 + pyproject.toml | 2 +- 6 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 examples/hermes-plugin/README.md create mode 100644 examples/hermes-plugin/pyproject.toml create mode 100644 examples/hermes-plugin/src/lmnr_hermes/__init__.py create mode 100644 examples/hermes-plugin/src/lmnr_hermes/plugin.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 2408f1ab..5e2b1f2c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,6 +62,13 @@ lmnr datasets pull # Pull dataset - All entries in the `[dependency-groups].dev` section of `pyproject.toml` MUST be pinned to a specific version with `==X.Y.Z`. Do NOT use unbounded specifiers (`>=`, `^`, `~`, `<`, ranges, or bare package names). Pinning keeps the test matrix deterministic across developers and CI. When adding a new dev dep, look up the current release on https://pypi.org and pin to that exact version; bumps then go through a normal PR. +## examples/hermes-plugin + +- Standalone pip package that bridges [Hermes Agent](https://github.com/nousresearch/hermes-agent) plugin hooks to Laminar spans. Ships via the `hermes_agent.plugins` entry-point group (`lmnr-hermes = "lmnr_hermes:register"`), so `pip install -e examples/hermes-plugin` plus `hermes plugins enable lmnr-hermes` is enough — no pip publish needed. +- The `examples/hermes-plugin` directory is a uv workspace member; adding a new example dir with `tool.uv.sources.lmnr = { workspace = true }` requires updating `[tool.uv.workspace].members` in the root `pyproject.toml` or uv errors with "not a workspace member". +- Hermes calls hooks on different threads (ThreadPoolExecutor for concurrent tool calls; delegate/subagent workers). The plugin keeps a `session_id → turn span` map and parents tool spans via `parent_span_context=Laminar.get_laminar_span_context_dict(turn_span)` rather than relying on the OTel current context, which is thread-local. +- Laminar writes session_id as the attribute `lmnr.association.properties.session_id` (prefix = `ASSOCIATION_PROPERTIES` constant). Tests asserting session scoping must check this exact key, not `session_id` or `SESSION_ID`. + ## Environment Variables ``` diff --git a/examples/hermes-plugin/README.md b/examples/hermes-plugin/README.md new file mode 100644 index 00000000..ed458f02 --- /dev/null +++ b/examples/hermes-plugin/README.md @@ -0,0 +1,52 @@ +# lmnr-hermes + +Laminar tracing plugin for [Hermes Agent](https://github.com/nousresearch/hermes-agent). + +Emits nested OpenTelemetry spans for every Hermes conversation turn, tool call, and +subagent delegation — and lets Laminar's raw-provider instrumentors (OpenAI, +Anthropic, Bedrock) nest their own GenAI spans under each turn for free. + +## What gets traced + +Per turn (one Hermes `run_conversation` call): + +- `hermes.turn` (root span) — input: user message + model/platform; output: + assistant response. Session and user IDs are attached. +- `tool.` (child, `span_type=TOOL`) — one per tool call, with args, + result, and `duration_ms`. +- `subagent.` (child) — one per finished subagent delegation. +- OpenAI / Anthropic / Bedrock spans — nested under the turn automatically, + with token usage and messages. + +Span attributes include `hermes.model`, `hermes.provider`, `hermes.api_mode`, +`hermes.finish_reason`, `hermes.usage.*`, and `hermes.tool_duration_ms`. + +## Install (local dev, no pip publish) + +```bash +# 1. Editable install — exposes the hermes_agent.plugins entry point +pip install -e path/to/lmnr-python/examples/hermes-plugin + +# 2. Enable in Hermes +hermes plugins enable lmnr-hermes + +# 3. Set your Laminar project key and run +export LMNR_PROJECT_API_KEY=... +hermes run +``` + +Alternative: drop the `src/lmnr_hermes/` directory as `~/.hermes/plugins/lmnr-hermes/` +and Hermes will discover it on startup (the `plugin.yaml` manifest ships inside +that directory). Then `hermes plugins enable lmnr-hermes`. + +## Configuration + +The plugin reads standard Laminar environment variables: + +| Variable | Purpose | +|------------------------|---------------------------------------------------| +| `LMNR_PROJECT_API_KEY` | Your project key (required unless OTel env is set) | +| `LMNR_BASE_URL` | Override the Laminar endpoint (default: api.lmnr.ai) | + +If the key is missing, the plugin no-ops silently — Hermes keeps working +without tracing. diff --git a/examples/hermes-plugin/pyproject.toml b/examples/hermes-plugin/pyproject.toml new file mode 100644 index 00000000..d8fa5001 --- /dev/null +++ b/examples/hermes-plugin/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "lmnr-hermes" +version = "0.1.0" +description = "Laminar tracing plugin for Hermes Agent (nousresearch/hermes-agent)" +readme = "README.md" +requires-python = ">=3.10,<4" +license = "Apache-2.0" +authors = [ + { name = "lmnr.ai", email = "founders@lmnr.ai" }, +] +dependencies = [ + "lmnr>=0.7.0", +] + +[project.entry-points."hermes_agent.plugins"] +lmnr-hermes = "lmnr_hermes:register" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/lmnr_hermes"] + +[tool.uv.sources] +lmnr = { workspace = true } diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py new file mode 100644 index 00000000..2b07066e --- /dev/null +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -0,0 +1,411 @@ +"""Laminar tracing plugin for Hermes Agent. + +Wires Hermes's plugin hooks (``pre_llm_call``, ``pre_tool_call``, +``post_tool_call``, ``on_session_end``, ``subagent_stop``) into Laminar +spans so every conversation turn, tool call, and subagent run shows up +as a proper nested trace in the Laminar UI. + +Installation (local dev, no pip publish needed):: + + pip install -e path/to/lmnr-python/examples/hermes-plugin + hermes plugins enable lmnr-hermes + +Or drop the package under ``~/.hermes/plugins/lmnr-hermes/`` and the +directory loader will pick it up. + +Initialization reads ``LMNR_PROJECT_API_KEY`` / ``LMNR_BASE_URL`` from the +environment. Raw provider instrumentors (OpenAI, Anthropic, Bedrock) are +auto-enabled by :func:`lmnr.Laminar.initialize` and will nest their spans +under the current Hermes turn automatically. +""" + +from __future__ import annotations + +import logging +import os +import threading +from typing import Any + +from lmnr import Laminar +from lmnr.sdk.types import LaminarSpanContext +from opentelemetry import context as context_api +from opentelemetry import trace + +logger = logging.getLogger(__name__) + +_init_lock = threading.Lock() +_initialized = False + +# (session_id, tool_call_id) -> active tool span. Entries are inserted in +# pre_tool_call and popped in post_tool_call. Same-thread access within a +# single tool dispatch, so no extra locking needed. +_tool_spans: dict[tuple[str, str], Any] = {} + +# session_id -> {"span": LaminarSpan, "ctx_token": OTel context token, +# "lmnr_ctx": LaminarSpanContext serialized for cross-thread reuse} +_turn_state: dict[str, dict[str, Any]] = {} +_turn_lock = threading.Lock() + + +def _ensure_initialized() -> bool: + """Initialize Laminar once. Returns True if tracing is active.""" + global _initialized + if _initialized: + return True + with _init_lock: + if _initialized: + return True + if Laminar.is_initialized(): + _initialized = True + return True + api_key = os.environ.get("LMNR_PROJECT_API_KEY") + if not api_key and not os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"): + logger.info( + "lmnr-hermes: LMNR_PROJECT_API_KEY not set; tracing disabled." + ) + return False + try: + Laminar.initialize() + except Exception as exc: + logger.warning("lmnr-hermes: Laminar.initialize() failed: %s", exc) + return False + _initialized = True + return True + + +def _safe_str(value: Any, limit: int = 4000) -> str: + try: + s = value if isinstance(value, str) else str(value) + except Exception: + return "" + return s if len(s) <= limit else s[:limit] + "...[truncated]" + + +# --------------------------------------------------------------------------- +# Hook callbacks +# --------------------------------------------------------------------------- + + +def _on_session_start( + session_id: str = "", + model: str = "", + platform: str = "", + **_: Any, +) -> None: + if not _ensure_initialized(): + return + # Nothing to span here — the first real work is pre_llm_call on the same + # turn. Keep this hook registered so downstream metadata (e.g. platform) + # is available for logging and so future plugin revisions can use it. + logger.debug( + "lmnr-hermes: session start session_id=%s model=%s platform=%s", + session_id, model, platform, + ) + + +def _on_pre_llm_call( + session_id: str = "", + user_message: str = "", + conversation_history: list | None = None, + is_first_turn: bool = False, + model: str = "", + platform: str = "", + sender_id: str = "", + **_: Any, +) -> None: + if not _ensure_initialized(): + return + if not session_id: + return + + # If a prior turn's span was never closed (on_session_end missed, crash, + # etc.), close it before starting a new one so we don't leak spans. + _close_turn(session_id, reason="reenter") + + try: + span = Laminar.start_span( + name="hermes.turn", + input={ + "user_message": _safe_str(user_message), + "model": model, + "platform": platform, + "is_first_turn": is_first_turn, + "history_len": len(conversation_history or []), + }, + session_id=session_id, + user_id=sender_id or None, + tags=[t for t in ["hermes", platform or None] if t], + attributes={ + "hermes.model": model or "", + "hermes.platform": platform or "", + "hermes.is_first_turn": bool(is_first_turn), + }, + ) + except Exception as exc: + logger.warning("lmnr-hermes: failed to start turn span: %s", exc) + return + + # Attach span to the OTel current context so raw provider instrumentors + # (OpenAI, Anthropic, Bedrock) running on the main thread parent under it. + try: + ctx = trace.set_span_in_context(span) + ctx_token = context_api.attach(ctx) + except Exception as exc: + logger.warning("lmnr-hermes: attach context failed: %s", exc) + ctx_token = None + + try: + lmnr_ctx = Laminar.get_laminar_span_context_dict(span) + except Exception: + lmnr_ctx = None + + with _turn_lock: + _turn_state[session_id] = { + "span": span, + "ctx_token": ctx_token, + "lmnr_ctx": lmnr_ctx, + } + + +def _on_post_api_request( + task_id: str = "", + session_id: str = "", + platform: str = "", + model: str = "", + provider: str = "", + api_mode: str = "", + api_call_count: int = 0, + api_duration: float | None = None, + finish_reason: str = "", + usage: dict | None = None, + response_model: str | None = None, + assistant_content_chars: int = 0, + assistant_tool_call_count: int = 0, + **_: Any, +) -> None: + """Attach per-API-call attributes to the turn span so the UI shows token + usage and finish reason even when provider instrumentors are disabled.""" + if not _ensure_initialized(): + return + with _turn_lock: + state = _turn_state.get(session_id) + if not state: + return + span = state["span"] + try: + attrs: dict[str, Any] = { + "hermes.provider": provider or "", + "hermes.api_mode": api_mode or "", + "hermes.api_call_count": int(api_call_count or 0), + "hermes.finish_reason": finish_reason or "", + "hermes.assistant_tool_call_count": int(assistant_tool_call_count or 0), + } + if api_duration is not None: + attrs["hermes.api_duration_ms"] = int(float(api_duration) * 1000) + if response_model: + attrs["hermes.response_model"] = response_model + if isinstance(usage, dict): + for k, v in usage.items(): + if isinstance(v, (int, float, str, bool)): + attrs[f"hermes.usage.{k}"] = v + for k, v in attrs.items(): + span.set_attribute(k, v) + except Exception as exc: + logger.debug("lmnr-hermes: set post_api_request attrs failed: %s", exc) + + +def _on_pre_tool_call( + tool_name: str = "", + args: dict | None = None, + task_id: str = "", + session_id: str = "", + tool_call_id: str = "", + **_: Any, +) -> None: + if not _ensure_initialized(): + return + if not session_id or not tool_call_id: + return + + with _turn_lock: + state = _turn_state.get(session_id) + parent_ctx = state.get("lmnr_ctx") if state else None + + try: + span = Laminar.start_span( + name=f"tool.{tool_name}" if tool_name else "tool", + input={"tool_name": tool_name, "args": args or {}}, + span_type="TOOL", + parent_span_context=parent_ctx, + session_id=session_id or None, + attributes={ + "hermes.tool_name": tool_name or "", + "hermes.tool_call_id": tool_call_id or "", + "hermes.task_id": task_id or "", + }, + ) + except Exception as exc: + logger.debug("lmnr-hermes: start tool span failed: %s", exc) + return + + _tool_spans[(session_id, tool_call_id)] = span + + +def _on_post_tool_call( + tool_name: str = "", + args: dict | None = None, + result: Any = None, + task_id: str = "", + session_id: str = "", + tool_call_id: str = "", + duration_ms: int = 0, + **_: Any, +) -> None: + if not _ensure_initialized(): + return + span = _tool_spans.pop((session_id, tool_call_id), None) + if span is None: + return + try: + span.set_attribute("hermes.tool_duration_ms", int(duration_ms or 0)) + span.set_output(_safe_str(result, limit=16000)) + except Exception: + pass + try: + span.end() + except Exception: + pass + + +def _close_turn(session_id: str, *, reason: str = "session_end") -> None: + with _turn_lock: + state = _turn_state.pop(session_id, None) + if not state: + return + span = state.get("span") + ctx_token = state.get("ctx_token") + if ctx_token is not None: + try: + context_api.detach(ctx_token) + except Exception: + pass + if span is None: + return + try: + span.set_attribute("hermes.turn_end_reason", reason) + except Exception: + pass + try: + span.end() + except Exception: + pass + + +def _on_post_llm_call( + session_id: str = "", + user_message: str = "", + assistant_response: str = "", + conversation_history: list | None = None, + model: str = "", + platform: str = "", + **_: Any, +) -> None: + if not _ensure_initialized(): + return + with _turn_lock: + state = _turn_state.get(session_id) + if not state: + return + span = state["span"] + try: + span.set_output(_safe_str(assistant_response, limit=16000)) + except Exception: + pass + + +def _on_session_end( + session_id: str = "", + completed: bool = True, + interrupted: bool = False, + model: str = "", + platform: str = "", + **_: Any, +) -> None: + if not _ensure_initialized(): + return + with _turn_lock: + state = _turn_state.get(session_id) + if state is not None: + span = state.get("span") + try: + if span is not None: + span.set_attribute("hermes.completed", bool(completed)) + span.set_attribute("hermes.interrupted", bool(interrupted)) + except Exception: + pass + _close_turn( + session_id, + reason="interrupted" if interrupted else ("completed" if completed else "ended"), + ) + + +def _on_subagent_stop( + parent_session_id: str = "", + child_role: str = "", + child_summary: str = "", + child_status: str = "", + duration_ms: int = 0, + **_: Any, +) -> None: + """Emit a retrospective span for a finished subagent delegation. + + Hermes fires this on the parent thread after the child agent has already + finished, so we can't span the child's execution itself — we just record + a short instantaneous span with the child's metadata nested under the + parent's turn span. + """ + if not _ensure_initialized(): + return + with _turn_lock: + state = _turn_state.get(parent_session_id) + parent_ctx = state.get("lmnr_ctx") if state else None + try: + span = Laminar.start_span( + name=f"subagent.{child_role}" if child_role else "subagent", + input={"role": child_role}, + parent_span_context=parent_ctx, + session_id=parent_session_id or None, + attributes={ + "hermes.subagent.role": child_role or "", + "hermes.subagent.status": child_status or "", + "hermes.subagent.duration_ms": int(duration_ms or 0), + }, + ) + span.set_output(_safe_str(child_summary, limit=8000)) + span.end() + except Exception as exc: + logger.debug("lmnr-hermes: subagent span failed: %s", exc) + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def register(ctx) -> None: + """Plugin entry point called by Hermes's PluginManager.""" + # Initialize eagerly so the Laminar SDK can start its background exporter + # before the first turn — cleaner startup logs than lazy-initializing in + # the first hook call. + _ensure_initialized() + ctx.register_hook("on_session_start", _on_session_start) + ctx.register_hook("pre_llm_call", _on_pre_llm_call) + ctx.register_hook("post_api_request", _on_post_api_request) + ctx.register_hook("pre_tool_call", _on_pre_tool_call) + ctx.register_hook("post_tool_call", _on_post_tool_call) + ctx.register_hook("post_llm_call", _on_post_llm_call) + ctx.register_hook("on_session_end", _on_session_end) + ctx.register_hook("subagent_stop", _on_subagent_stop) + + +__all__ = ["register"] diff --git a/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml b/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml new file mode 100644 index 00000000..65ba6c85 --- /dev/null +++ b/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml @@ -0,0 +1,17 @@ +name: lmnr-hermes +version: 0.1.0 +description: "Laminar tracing plugin for Hermes Agent — emits nested spans for every turn, tool call, and subagent." +author: "lmnr.ai" +kind: standalone +requires_env: + - name: LMNR_PROJECT_API_KEY + description: "Laminar project API key (get one at https://www.lmnr.ai)." +provides_hooks: + - on_session_start + - pre_llm_call + - post_api_request + - pre_tool_call + - post_tool_call + - post_llm_call + - on_session_end + - subagent_stop diff --git a/pyproject.toml b/pyproject.toml index 2a8456b1..40c0cd29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,7 +144,7 @@ requires = ["uv_build>=0.11.7,<0.12"] build-backend = "uv_build" [tool.uv.workspace] -members = ["examples/fastapi-app"] +members = ["examples/fastapi-app", "examples/hermes-plugin"] [tool.ruff] target-version = "py310" From 2c007cd1396b20920e314b5983bd46b75a72b573 Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 16:48:46 +0000 Subject: [PATCH 2/9] fix(hermes-plugin): cache failed initialization to avoid retry overhead Co-Authored-By: Claude Opus 4.7 --- .../hermes-plugin/src/lmnr_hermes/__init__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index 2b07066e..e7c1d2fa 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -35,6 +35,7 @@ _init_lock = threading.Lock() _initialized = False +_init_failed = False # (session_id, tool_call_id) -> active tool span. Entries are inserted in # pre_tool_call and popped in post_tool_call. Same-thread access within a @@ -48,13 +49,22 @@ def _ensure_initialized() -> bool: - """Initialize Laminar once. Returns True if tracing is active.""" - global _initialized + """Initialize Laminar once. Returns True if tracing is active. + + A failed initialization (missing key or exception) is cached in + ``_init_failed`` so subsequent hook invocations short-circuit without + re-acquiring the lock or re-running expensive setup. + """ + global _initialized, _init_failed if _initialized: return True + if _init_failed: + return False with _init_lock: if _initialized: return True + if _init_failed: + return False if Laminar.is_initialized(): _initialized = True return True @@ -63,11 +73,13 @@ def _ensure_initialized() -> bool: logger.info( "lmnr-hermes: LMNR_PROJECT_API_KEY not set; tracing disabled." ) + _init_failed = True return False try: Laminar.initialize() except Exception as exc: logger.warning("lmnr-hermes: Laminar.initialize() failed: %s", exc) + _init_failed = True return False _initialized = True return True From 289abbb498fed2ac28b570fddf1b937cc492ee5a Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 16:51:30 +0000 Subject: [PATCH 3/9] fix(hermes-plugin): atomically pop turn state in on_session_end Avoid a TOCTOU race where a concurrent pre_llm_call could replace the session's turn state between the get() and the pop() in _close_turn, causing the newly created turn span to be ended instead of the original. Co-Authored-By: Claude Opus 4.7 --- .../hermes-plugin/src/lmnr_hermes/__init__.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index e7c1d2fa..f4edb8b2 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -289,9 +289,15 @@ def _on_post_tool_call( pass -def _close_turn(session_id: str, *, reason: str = "session_end") -> None: - with _turn_lock: - state = _turn_state.pop(session_id, None) +def _finish_turn_state( + state: dict[str, Any] | None, *, reason: str +) -> None: + """End a turn span from an already-popped state entry. + + Split out so callers that need to atomically pop-and-mutate the state + (e.g. ``_on_session_end``) can do so under a single lock acquisition + instead of a get-then-pop that another thread could interleave. + """ if not state: return span = state.get("span") @@ -313,6 +319,12 @@ def _close_turn(session_id: str, *, reason: str = "session_end") -> None: pass +def _close_turn(session_id: str, *, reason: str = "session_end") -> None: + with _turn_lock: + state = _turn_state.pop(session_id, None) + _finish_turn_state(state, reason=reason) + + def _on_post_llm_call( session_id: str = "", user_message: str = "", @@ -345,8 +357,10 @@ def _on_session_end( ) -> None: if not _ensure_initialized(): return + # Atomically pop the state so a concurrent pre_llm_call that installs a + # new turn for this session_id can't have its state ended here by mistake. with _turn_lock: - state = _turn_state.get(session_id) + state = _turn_state.pop(session_id, None) if state is not None: span = state.get("span") try: @@ -355,8 +369,8 @@ def _on_session_end( span.set_attribute("hermes.interrupted", bool(interrupted)) except Exception: pass - _close_turn( - session_id, + _finish_turn_state( + state, reason="interrupted" if interrupted else ("completed" if completed else "ended"), ) From b6af91377b2154e0d092688e78a1daa71e621d12 Mon Sep 17 00:00:00 2001 From: Robert Kim Date: Mon, 27 Apr 2026 19:44:47 +0000 Subject: [PATCH 4/9] feat(hermes-plugin): emit LLM spans from pre/post_api_request hooks Hermes drives its own HTTP client for provider calls (see run_agent.py ~L11880 _get_transport().normalize_response), not the anthropic / openai Python SDKs, so Laminar's auto-enabled raw-SDK instrumentors never see these calls. Without direct emission the Laminar UI showed tool spans but no model spans and reported $0 cost per trace. Open an LLM span (span_type="LLM") in pre_api_request parented to the turn's LaminarSpanContext, and close it in post_api_request after setting the GenAI semconv usage attributes. Map Hermes's CanonicalUsage fields (input_tokens / output_tokens / cache_read_tokens / cache_write_tokens / reasoning_tokens) onto the exact attribute keys Laminar reads for cost computation (gen_ai.usage.cache_read_input_tokens, gen_ai.usage.cache_creation_input_tokens). Sweep any dangling api spans in _close_turn / _on_session_end via _cleanup_api_spans so a provider error between pre_ and post_ can't leak a span. Also honour LMNR_HTTP_PORT / LMNR_GRPC_PORT env vars in the plugin's _ensure_initialized so a local Laminar dev instance (gRPC 8001, not the SDK default 8443) can be targeted without having to pass ports through the base URL (which the SDK strips). Fix the entry_points declaration to point at the module (`lmnr_hermes`) rather than the register callable; Hermes's loader does ep.load() then looks for a `register` attribute on the result. Co-Authored-By: Claude Opus 4.7 --- CLAUDE.md | 3 + examples/hermes-plugin/pyproject.toml | 5 +- .../hermes-plugin/src/lmnr_hermes/__init__.py | 177 ++++++++++++++++-- 3 files changed, 169 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5e2b1f2c..87bd629b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,6 +68,9 @@ lmnr datasets pull # Pull dataset - The `examples/hermes-plugin` directory is a uv workspace member; adding a new example dir with `tool.uv.sources.lmnr = { workspace = true }` requires updating `[tool.uv.workspace].members` in the root `pyproject.toml` or uv errors with "not a workspace member". - Hermes calls hooks on different threads (ThreadPoolExecutor for concurrent tool calls; delegate/subagent workers). The plugin keeps a `session_id → turn span` map and parents tool spans via `parent_span_context=Laminar.get_laminar_span_context_dict(turn_span)` rather than relying on the OTel current context, which is thread-local. - Laminar writes session_id as the attribute `lmnr.association.properties.session_id` (prefix = `ASSOCIATION_PROPERTIES` constant). Tests asserting session scoping must check this exact key, not `session_id` or `SESSION_ID`. +- Hermes drives its own HTTP client for provider calls (see `run_agent.py` ~line 11880 using `_get_transport().normalize_response`), NOT the `anthropic` / `openai` Python SDKs. The raw-SDK instrumentors Laminar auto-enables therefore never see these calls. The plugin must create LLM spans itself in the `pre_api_request` hook (open) and `post_api_request` hook (close + usage), using `span_type="LLM"` and the GenAI semconv attributes (`gen_ai.system`, `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.cache_read_input_tokens`, `gen_ai.usage.cache_creation_input_tokens`). Without these, traces show tool spans but no model spans, and the Laminar UI reports `$0` cost. +- The `post_api_request` hook's `usage` payload is the dict form of Hermes's `CanonicalUsage` dataclass (`agent/usage_pricing.py`): fields are `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `total_tokens`. Map `cache_read_tokens` → `gen_ai.usage.cache_read_input_tokens` and `cache_write_tokens` → `gen_ai.usage.cache_creation_input_tokens` so Laminar's cost computation matches Anthropic's pricing tiers. +- Laminar's SDK strips the port from `LMNR_BASE_URL` ("Ignoring port in base URL: 8000"). To point the plugin at a local dev instance, pass `http_port` / `grpc_port` to `Laminar.initialize()`. The plugin reads `LMNR_HTTP_PORT` and `LMNR_GRPC_PORT` env vars for this. Local app-server gRPC is `8001`, not the SDK's default `8443`. ## Environment Variables diff --git a/examples/hermes-plugin/pyproject.toml b/examples/hermes-plugin/pyproject.toml index d8fa5001..37e7dbfe 100644 --- a/examples/hermes-plugin/pyproject.toml +++ b/examples/hermes-plugin/pyproject.toml @@ -13,7 +13,10 @@ dependencies = [ ] [project.entry-points."hermes_agent.plugins"] -lmnr-hermes = "lmnr_hermes:register" +# Hermes's entry-point loader calls `ep.load()` then looks for a `register` +# attribute on the result, so the entry must resolve to the *module*, not the +# register function itself. +lmnr-hermes = "lmnr_hermes" [build-system] requires = ["hatchling"] diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index f4edb8b2..be26515d 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -42,6 +42,12 @@ # single tool dispatch, so no extra locking needed. _tool_spans: dict[tuple[str, str], Any] = {} +# (session_id, api_call_count) -> active LLM span. Opened in pre_api_request +# and closed in post_api_request. Hermes drives its own HTTP client so the +# raw anthropic/openai instrumentors never see these calls — we have to emit +# LLM spans ourselves to get token usage and cost into the Laminar UI. +_api_spans: dict[tuple[str, int], Any] = {} + # session_id -> {"span": LaminarSpan, "ctx_token": OTel context token, # "lmnr_ctx": LaminarSpanContext serialized for cross-thread reuse} _turn_state: dict[str, dict[str, Any]] = {} @@ -75,8 +81,15 @@ def _ensure_initialized() -> bool: ) _init_failed = True return False + init_kwargs: dict[str, Any] = {} + http_port = os.environ.get("LMNR_HTTP_PORT") + grpc_port = os.environ.get("LMNR_GRPC_PORT") + if http_port: + init_kwargs["http_port"] = int(http_port) + if grpc_port: + init_kwargs["grpc_port"] = int(grpc_port) try: - Laminar.initialize() + Laminar.initialize(**init_kwargs) except Exception as exc: logger.warning("lmnr-hermes: Laminar.initialize() failed: %s", exc) _init_failed = True @@ -179,6 +192,78 @@ def _on_pre_llm_call( } +def _on_pre_api_request( + task_id: str = "", + session_id: str = "", + platform: str = "", + model: str = "", + provider: str = "", + base_url: str = "", + api_mode: str = "", + api_call_count: int = 0, + message_count: int = 0, + tool_count: int = 0, + approx_input_tokens: int = 0, + request_char_count: int = 0, + max_tokens: int | None = None, + **_: Any, +) -> None: + """Open an LLM span for each provider API call. + + Hermes drives its own HTTP client (not the ``anthropic`` / ``openai`` + Python SDKs) so the raw-SDK instrumentors Laminar auto-enables never see + these calls. We emit the span ourselves and close it in + ``post_api_request`` where the response usage dict is available. + """ + if not _ensure_initialized(): + return + if not session_id: + return + + with _turn_lock: + state = _turn_state.get(session_id) + parent_ctx = state.get("lmnr_ctx") if state else None + + # Short human-readable model slug for the span name + span_name = f"llm.{provider}" if provider else "llm" + if model: + span_name = f"{span_name}.{model}" + + try: + span = Laminar.start_span( + name=span_name, + span_type="LLM", + parent_span_context=parent_ctx, + session_id=session_id or None, + attributes={ + "gen_ai.system": provider or "", + "gen_ai.request.model": model or "", + "hermes.api_mode": api_mode or "", + "hermes.api_call_count": int(api_call_count or 0), + "hermes.task_id": task_id or "", + "hermes.message_count": int(message_count or 0), + "hermes.tool_count": int(tool_count or 0), + "hermes.approx_input_tokens": int(approx_input_tokens or 0), + "hermes.request_char_count": int(request_char_count or 0), + }, + ) + if base_url: + try: + span.set_attribute("gen_ai.request.server_address", base_url) + except Exception: + pass + if max_tokens is not None: + try: + span.set_attribute("gen_ai.request.max_tokens", int(max_tokens)) + except Exception: + pass + except Exception as exc: + logger.debug("lmnr-hermes: start llm span failed: %s", exc) + return + + _api_spans[(session_id, int(api_call_count or 0))] = span + + def _on_post_api_request( task_id: str = "", session_id: str = "", @@ -195,35 +280,72 @@ def _on_post_api_request( assistant_tool_call_count: int = 0, **_: Any, ) -> None: - """Attach per-API-call attributes to the turn span so the UI shows token - usage and finish reason even when provider instrumentors are disabled.""" + """Close the LLM span opened in pre_api_request with usage and output.""" if not _ensure_initialized(): return - with _turn_lock: - state = _turn_state.get(session_id) - if not state: + span = _api_spans.pop((session_id, int(api_call_count or 0)), None) + if span is None: return - span = state["span"] + try: + if isinstance(usage, dict): + # Laminar derives cost from these exact attribute keys (+ model). + input_tokens = int(usage.get("input_tokens") or 0) + output_tokens = int(usage.get("output_tokens") or 0) + cache_read = int(usage.get("cache_read_tokens") or 0) + cache_write = int(usage.get("cache_write_tokens") or 0) + reasoning = int(usage.get("reasoning_tokens") or 0) + total = int( + usage.get("total_tokens") + or (input_tokens + output_tokens + cache_read + cache_write) + ) + span.set_attribute("gen_ai.usage.input_tokens", input_tokens) + span.set_attribute("gen_ai.usage.output_tokens", output_tokens) + if cache_read: + span.set_attribute("gen_ai.usage.cache_read_input_tokens", cache_read) + if cache_write: + span.set_attribute( + "gen_ai.usage.cache_creation_input_tokens", cache_write + ) + if reasoning: + span.set_attribute("gen_ai.usage.reasoning_tokens", reasoning) + if total: + span.set_attribute("llm.usage.total_tokens", total) + attrs: dict[str, Any] = { - "hermes.provider": provider or "", - "hermes.api_mode": api_mode or "", - "hermes.api_call_count": int(api_call_count or 0), "hermes.finish_reason": finish_reason or "", + "hermes.assistant_content_chars": int(assistant_content_chars or 0), "hermes.assistant_tool_call_count": int(assistant_tool_call_count or 0), } if api_duration is not None: attrs["hermes.api_duration_ms"] = int(float(api_duration) * 1000) if response_model: + attrs["gen_ai.response.model"] = response_model attrs["hermes.response_model"] = response_model - if isinstance(usage, dict): - for k, v in usage.items(): - if isinstance(v, (int, float, str, bool)): - attrs[f"hermes.usage.{k}"] = v for k, v in attrs.items(): span.set_attribute(k, v) except Exception as exc: - logger.debug("lmnr-hermes: set post_api_request attrs failed: %s", exc) + logger.debug("lmnr-hermes: set llm span attrs failed: %s", exc) + + try: + # A brief summary is more useful than a raw JSON dump of the whole + # response body — we don't receive the body in the hook payload + # anyway. Surface finish reason + tool-call count as the "output". + output = { + "finish_reason": finish_reason or "", + "assistant_content_chars": int(assistant_content_chars or 0), + "assistant_tool_call_count": int(assistant_tool_call_count or 0), + } + if response_model: + output["response_model"] = response_model + span.set_output(output) + except Exception: + pass + + try: + span.end() + except Exception: + pass def _on_pre_tool_call( @@ -289,6 +411,28 @@ def _on_post_tool_call( pass +def _cleanup_api_spans(session_id: str) -> None: + """Close any LLM spans opened in ``pre_api_request`` but never closed. + + A provider call that raises before ``post_api_request`` fires would leave + the span dangling; sweep it when the turn ends so the trace is never + missing its closing event. + """ + leaked = [k for k in _api_spans if k[0] == session_id] + for k in leaked: + span = _api_spans.pop(k, None) + if span is None: + continue + try: + span.set_attribute("hermes.llm_end_reason", "turn_closed") + except Exception: + pass + try: + span.end() + except Exception: + pass + + def _finish_turn_state( state: dict[str, Any] | None, *, reason: str ) -> None: @@ -322,6 +466,7 @@ def _finish_turn_state( def _close_turn(session_id: str, *, reason: str = "session_end") -> None: with _turn_lock: state = _turn_state.pop(session_id, None) + _cleanup_api_spans(session_id) _finish_turn_state(state, reason=reason) @@ -369,6 +514,7 @@ def _on_session_end( span.set_attribute("hermes.interrupted", bool(interrupted)) except Exception: pass + _cleanup_api_spans(session_id) _finish_turn_state( state, reason="interrupted" if interrupted else ("completed" if completed else "ended"), @@ -426,6 +572,7 @@ def register(ctx) -> None: _ensure_initialized() ctx.register_hook("on_session_start", _on_session_start) ctx.register_hook("pre_llm_call", _on_pre_llm_call) + ctx.register_hook("pre_api_request", _on_pre_api_request) ctx.register_hook("post_api_request", _on_post_api_request) ctx.register_hook("pre_tool_call", _on_pre_tool_call) ctx.register_hook("post_tool_call", _on_post_tool_call) From fe06172904d17691320063361ca48f999789c13c Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 19:53:52 +0000 Subject: [PATCH 5/9] fix(hermes-plugin): sweep orphaned tool spans on turn end Mirror the _cleanup_api_spans logic for _tool_spans: concurrent tool calls on Hermes's ThreadPoolExecutor can raise before post_tool_call fires, leaking the span entry forever. Sweep by session_id when the turn closes or the session ends. Co-Authored-By: Claude Opus 4.7 --- .../hermes-plugin/src/lmnr_hermes/__init__.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index be26515d..77c3175e 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -433,6 +433,28 @@ def _cleanup_api_spans(session_id: str) -> None: pass +def _cleanup_tool_spans(session_id: str) -> None: + """Close any TOOL spans opened in ``pre_tool_call`` but never closed. + + Hermes runs concurrent tool calls on a ThreadPoolExecutor; an exception + in one worker will not fire ``post_tool_call`` for that call, leaking + its span in ``_tool_spans``. Sweep by session when the turn ends. + """ + leaked = [k for k in _tool_spans if k[0] == session_id] + for k in leaked: + span = _tool_spans.pop(k, None) + if span is None: + continue + try: + span.set_attribute("hermes.tool_end_reason", "turn_closed") + except Exception: + pass + try: + span.end() + except Exception: + pass + + def _finish_turn_state( state: dict[str, Any] | None, *, reason: str ) -> None: @@ -467,6 +489,7 @@ def _close_turn(session_id: str, *, reason: str = "session_end") -> None: with _turn_lock: state = _turn_state.pop(session_id, None) _cleanup_api_spans(session_id) + _cleanup_tool_spans(session_id) _finish_turn_state(state, reason=reason) @@ -515,6 +538,7 @@ def _on_session_end( except Exception: pass _cleanup_api_spans(session_id) + _cleanup_tool_spans(session_id) _finish_turn_state( state, reason="interrupted" if interrupted else ("completed" if completed else "ended"), From b6dc70506a3a48583aa89a9be699786f78abeda2 Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 19:56:08 +0000 Subject: [PATCH 6/9] fix(hermes-plugin): declare pre_api_request in plugin.yaml manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit register() wires nine hooks but the manifest only listed eight. Hermes's directory-based loader validates registered hooks against this list, so pre_api_request could silently fail to fire — and without it no LLM spans get opened, breaking the cost/usage display in the Laminar UI. Co-Authored-By: Claude Opus 4.7 --- examples/hermes-plugin/src/lmnr_hermes/plugin.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml b/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml index 65ba6c85..1f74b857 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml +++ b/examples/hermes-plugin/src/lmnr_hermes/plugin.yaml @@ -9,6 +9,7 @@ requires_env: provides_hooks: - on_session_start - pre_llm_call + - pre_api_request - post_api_request - pre_tool_call - post_tool_call From 30b8243e21af5c776cfed3bf36cdfbcdabbef867 Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 19:57:54 +0000 Subject: [PATCH 7/9] fix(hermes-plugin): preserve explicit zero total_tokens from provider Use an is-None check instead of `or` when reading total_tokens so a legitimate 0 (e.g. fully-cached or rejected request) is kept rather than silently replaced by a computed sum. Also always set the total_tokens attribute even when zero, so downstream consumers see the reported value. Co-Authored-By: Claude Opus 4.7 --- examples/hermes-plugin/src/lmnr_hermes/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index 77c3175e..70855884 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -295,10 +295,13 @@ def _on_post_api_request( cache_read = int(usage.get("cache_read_tokens") or 0) cache_write = int(usage.get("cache_write_tokens") or 0) reasoning = int(usage.get("reasoning_tokens") or 0) - total = int( - usage.get("total_tokens") - or (input_tokens + output_tokens + cache_read + cache_write) - ) + # Preserve an explicit provider-reported total (including 0); + # only fall back to the computed sum when the key is absent. + reported_total = usage.get("total_tokens") + if reported_total is not None: + total = int(reported_total) + else: + total = input_tokens + output_tokens + cache_read + cache_write span.set_attribute("gen_ai.usage.input_tokens", input_tokens) span.set_attribute("gen_ai.usage.output_tokens", output_tokens) if cache_read: @@ -309,8 +312,7 @@ def _on_post_api_request( ) if reasoning: span.set_attribute("gen_ai.usage.reasoning_tokens", reasoning) - if total: - span.set_attribute("llm.usage.total_tokens", total) + span.set_attribute("llm.usage.total_tokens", total) attrs: dict[str, Any] = { "hermes.finish_reason": finish_reason or "", From 98c79594f20af211e45a6abd97372429973f499a Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 20:08:58 +0000 Subject: [PATCH 8/9] fix(hermes-plugin): guard tool/api span dicts with a lock _cleanup_{api,tool}_spans iterated _api_spans / _tool_spans without a lock; a concurrent post_tool_call or post_api_request on another session could pop mid-iteration and raise "dictionary changed size during iteration", leaking the turn span and OTel context. Introduce _spans_lock and take it for all mutations so sweeping is safe under concurrency. Co-Authored-By: Claude Opus 4.7 --- .../hermes-plugin/src/lmnr_hermes/__init__.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index 70855884..b465621b 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -38,8 +38,9 @@ _init_failed = False # (session_id, tool_call_id) -> active tool span. Entries are inserted in -# pre_tool_call and popped in post_tool_call. Same-thread access within a -# single tool dispatch, so no extra locking needed. +# pre_tool_call and popped in post_tool_call. Accessed from Hermes's tool +# ThreadPoolExecutor workers and the turn thread, so all reads/writes go +# through ``_spans_lock``. _tool_spans: dict[tuple[str, str], Any] = {} # (session_id, api_call_count) -> active LLM span. Opened in pre_api_request @@ -48,6 +49,11 @@ # LLM spans ourselves to get token usage and cost into the Laminar UI. _api_spans: dict[tuple[str, int], Any] = {} +# Protects ``_tool_spans`` and ``_api_spans`` against concurrent mutation +# from multiple sessions / ThreadPoolExecutor workers. Separate from +# ``_turn_lock`` so turn-span reads don't contend with per-tool-call pops. +_spans_lock = threading.Lock() + # session_id -> {"span": LaminarSpan, "ctx_token": OTel context token, # "lmnr_ctx": LaminarSpanContext serialized for cross-thread reuse} _turn_state: dict[str, dict[str, Any]] = {} @@ -261,7 +267,8 @@ def _on_pre_api_request( logger.debug("lmnr-hermes: start llm span failed: %s", exc) return - _api_spans[(session_id, int(api_call_count or 0))] = span + with _spans_lock: + _api_spans[(session_id, int(api_call_count or 0))] = span def _on_post_api_request( @@ -283,7 +290,8 @@ def _on_post_api_request( """Close the LLM span opened in pre_api_request with usage and output.""" if not _ensure_initialized(): return - span = _api_spans.pop((session_id, int(api_call_count or 0)), None) + with _spans_lock: + span = _api_spans.pop((session_id, int(api_call_count or 0)), None) if span is None: return @@ -384,7 +392,8 @@ def _on_pre_tool_call( logger.debug("lmnr-hermes: start tool span failed: %s", exc) return - _tool_spans[(session_id, tool_call_id)] = span + with _spans_lock: + _tool_spans[(session_id, tool_call_id)] = span def _on_post_tool_call( @@ -399,7 +408,8 @@ def _on_post_tool_call( ) -> None: if not _ensure_initialized(): return - span = _tool_spans.pop((session_id, tool_call_id), None) + with _spans_lock: + span = _tool_spans.pop((session_id, tool_call_id), None) if span is None: return try: @@ -418,11 +428,15 @@ def _cleanup_api_spans(session_id: str) -> None: A provider call that raises before ``post_api_request`` fires would leave the span dangling; sweep it when the turn ends so the trace is never - missing its closing event. + missing its closing event. Snapshots + pops under ``_spans_lock`` so + concurrent ``post_api_request`` / ``post_tool_call`` calls from other + sessions can't mutate the dict mid-iteration. """ - leaked = [k for k in _api_spans if k[0] == session_id] - for k in leaked: - span = _api_spans.pop(k, None) + with _spans_lock: + leaked_spans = [ + _api_spans.pop(k) for k in list(_api_spans) if k[0] == session_id + ] + for span in leaked_spans: if span is None: continue try: @@ -440,11 +454,14 @@ def _cleanup_tool_spans(session_id: str) -> None: Hermes runs concurrent tool calls on a ThreadPoolExecutor; an exception in one worker will not fire ``post_tool_call`` for that call, leaking - its span in ``_tool_spans``. Sweep by session when the turn ends. + its span in ``_tool_spans``. Sweep by session when the turn ends, under + ``_spans_lock`` so concurrent workers can't mutate the dict mid-iteration. """ - leaked = [k for k in _tool_spans if k[0] == session_id] - for k in leaked: - span = _tool_spans.pop(k, None) + with _spans_lock: + leaked_spans = [ + _tool_spans.pop(k) for k in list(_tool_spans) if k[0] == session_id + ] + for span in leaked_spans: if span is None: continue try: From 3b25bcea4b37bc3838cc4f40ddd908d8eb7a2958 Mon Sep 17 00:00:00 2001 From: "cursor[bot]" Date: Mon, 27 Apr 2026 20:11:39 +0000 Subject: [PATCH 9/9] fix(hermes-plugin): include reasoning_tokens in total_tokens fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CanonicalUsage treats reasoning_tokens as a separate additive field, so when the provider omits total_tokens the computed fallback must include it — otherwise llm.usage.total_tokens is under-reported for reasoning models and skews Laminar's cost display. Co-Authored-By: Claude Opus 4.7 --- examples/hermes-plugin/src/lmnr_hermes/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/hermes-plugin/src/lmnr_hermes/__init__.py b/examples/hermes-plugin/src/lmnr_hermes/__init__.py index b465621b..b3025cbe 100644 --- a/examples/hermes-plugin/src/lmnr_hermes/__init__.py +++ b/examples/hermes-plugin/src/lmnr_hermes/__init__.py @@ -309,7 +309,16 @@ def _on_post_api_request( if reported_total is not None: total = int(reported_total) else: - total = input_tokens + output_tokens + cache_read + cache_write + # CanonicalUsage lists reasoning_tokens as a separate additive + # field, so include it in the fallback sum to avoid under- + # reporting when the provider omits total_tokens. + total = ( + input_tokens + + output_tokens + + cache_read + + cache_write + + reasoning + ) span.set_attribute("gen_ai.usage.input_tokens", input_tokens) span.set_attribute("gen_ai.usage.output_tokens", output_tokens) if cache_read: