From 62d4e8f07d5cab70c3861eb8efd3ebdf64ef068c Mon Sep 17 00:00:00 2001 From: drew Date: Sun, 17 May 2026 16:47:54 -0400 Subject: [PATCH] fix(telemetry): cost dashboard now computes real USD totals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three intertwined bugs caused every Cost-tab row to display \$0 even after the scraper started writing real token data: 1. **Lookup key mismatch.** ``_cost_usd`` looked up bare ``model`` but ``_DEFAULT_PRICES`` was keyed by ``provider/model`` — every priced model silently missed. Fixed by adding ``_price_key`` and a fallback chain: ``provider/model`` → bare ``model`` → ``_unknown``. 2. **SQL grouped by model only.** Same modelID served by two providers (e.g. ``claude-opus-4-6`` via Anthropic direct vs a local proxy) at different rates was conflated into one row. Fixed: ``GROUP BY model, provider`` in ``_api_cost`` + ``provider`` returned in each row. 3. **Math convention mismatch.** ``_cost_usd`` did ``(tokens_in - cached) * in_rate`` assuming ``tokens_in`` was total input. But the scraper records ``tokens_in`` as OpenCode's ``info.tokens.input`` (fresh, non-cached), so ``tokens_in - cached`` went negative whenever cache reads exceeded fresh input — which is the common case with Anthropic prompt caching. Fixed: no subtraction; the three populations bill at their three rates. Pricing seeded for the 8 models the scraper has actually observed (``_DEFAULT_PRICES`` corrected from stale Opus-3 numbers + new entries for the Haiku 4.5 / GPT-5 family / CleverThis HF endpoints): | Provider | Model | in | out | cached_in | |--------------|---------------------------|-------|-------|-----------| | local-claude | claude-opus-4-6 | 5.00 | 25.00 | 0.50 | | local-claude | claude-sonnet-4-6 | 3.00 | 15.00 | 0.30 | | local-claude | claude-haiku-4-5 | 1.00 | 5.00 | 0.10 | | openai | gpt-5 / gpt-5-codex | 1.25 | 10.00 | 0.125 | | openai | gpt-5-mini | 0.25 | 2.00 | 0.025 | | openai | gpt-5-nano | 0.05 | 0.40 | 0.005 | | CleverThis-* | (HF endpoints, advisory) | 0.50 | 1.00 | — | Operators can override without touching code via ``.opencode/telemetry/prices.json`` (added; same keying convention). ``_load_prices`` now skips ``_comment`` / ``_last_updated`` / ``_sources`` metadata keys so docs in the JSON don't pollute the table. Smoke against current ``llm_activity`` (3743 turns, 450 archives): total USD over the lifetime window is now \$118.06, with all 8 models showing as priced. Tests pin all three regressions: provider-qualified lookup, bare-model fallback, no-subtraction math, GROUP BY (model, provider), and the metadata-key filter in ``_load_prices``. Co-Authored-By: Claude Opus 4.7 (1M context) --- .opencode/telemetry/prices.json | 23 +++ .opencode/telemetry/server.py | 87 ++++++++--- tests/auto_agents/test_telemetry_server.py | 159 +++++++++++++++++++++ 3 files changed, 251 insertions(+), 18 deletions(-) create mode 100644 .opencode/telemetry/prices.json diff --git a/.opencode/telemetry/prices.json b/.opencode/telemetry/prices.json new file mode 100644 index 000000000..85bbe471f --- /dev/null +++ b/.opencode/telemetry/prices.json @@ -0,0 +1,23 @@ +{ + "_comment": "Telemetry token-cost overrides. Keys are 'provider/model' (matching OpenCode's info.providerID + info.modelID); values are USD per million tokens. The 'cached_in' field is optional and defaults to the 'in' rate when absent. See .opencode/telemetry/server.py:_DEFAULT_PRICES for the seeded fallback table.", + "_last_updated": "2026-05-17", + "_sources": { + "anthropic": "platform.claude.com/docs/en/about-claude/pricing (May 2026)", + "openai": "openai.com/api/pricing + pricepertoken.com (May 2026)", + "CleverThis-*": "HF Inference Endpoints — billed per endpoint-hour; per-token figures are imputed for dashboard reporting" + }, + + "local-claude/claude-opus-4-6": { "in": 5.0, "out": 25.0, "cached_in": 0.50 }, + "local-claude/claude-sonnet-4-6": { "in": 3.0, "out": 15.0, "cached_in": 0.30 }, + "local-claude/claude-haiku-4-5": { "in": 1.0, "out": 5.0, "cached_in": 0.10 }, + + "openai/gpt-5": { "in": 1.25, "out": 10.0, "cached_in": 0.125 }, + "openai/gpt-5-codex": { "in": 1.25, "out": 10.0, "cached_in": 0.125 }, + "openai/gpt-5-mini": { "in": 0.25, "out": 2.0, "cached_in": 0.025 }, + "openai/gpt-5-nano": { "in": 0.05, "out": 0.4, "cached_in": 0.005 }, + + "CleverThis-4/Kimi-K2-6-GGUF-Q2-K-XL": { "in": 0.5, "out": 1.0 }, + "CleverThis-15/Qwen3-6-35B-A3B-GGUF-UD-Q3-K-XL": { "in": 0.5, "out": 1.0 }, + "CleverThis/Qwen3-6-35B-A3B-GGUF-BF16": { "in": 0.5, "out": 1.0 }, + "CleverThis-2/MiniMax-M2-7-GGUF-BF16": { "in": 0.5, "out": 1.0 } +} diff --git a/.opencode/telemetry/server.py b/.opencode/telemetry/server.py index 141c5a50a..4e4bbb7bd 100644 --- a/.opencode/telemetry/server.py +++ b/.opencode/telemetry/server.py @@ -382,16 +382,26 @@ def _heartbeat_age(candidates: list[str]) -> dict[str, Any] | None: # Defaults seeded from publicly-listed prices; override via # .opencode/telemetry/prices.json. Costs are $/1,000,000 tokens. _DEFAULT_PRICES: dict[str, dict[str, float]] = { - # Anthropic (public) - "anthropic/claude-opus-4-6": {"in": 15.0, "out": 75.0, "cached_in": 1.5}, - "anthropic/claude-sonnet-4-6": {"in": 3.0, "out": 15.0, "cached_in": 0.3}, - # CleverThis HF Inference Endpoints — these are billed per - # endpoint-hour, not per token, so the per-token estimate here is a - # best-effort imputation for "what would this have cost on a public - # API"; treat as advisory not source-of-truth. + # Anthropic (public — May 2026 list prices). Cache reads are + # 10% of standard input across the family. + "local-claude/claude-opus-4-6": {"in": 5.0, "out": 25.0, "cached_in": 0.50}, + "local-claude/claude-sonnet-4-6": {"in": 3.0, "out": 15.0, "cached_in": 0.30}, + "local-claude/claude-haiku-4-5": {"in": 1.0, "out": 5.0, "cached_in": 0.10}, + # OpenAI (public — May 2026 list prices). gpt-5 / gpt-5-codex + # share the same per-token rate; gpt-5-mini / gpt-5-nano are + # the budget tier. + "openai/gpt-5": {"in": 1.25, "out": 10.0, "cached_in": 0.125}, + "openai/gpt-5-codex": {"in": 1.25, "out": 10.0, "cached_in": 0.125}, + "openai/gpt-5-mini": {"in": 0.25, "out": 2.0, "cached_in": 0.025}, + "openai/gpt-5-nano": {"in": 0.05, "out": 0.4, "cached_in": 0.005}, + # CleverThis HF Inference Endpoints — billed per endpoint-hour, + # not per token. The per-token entries here are a best-effort + # imputation for "what would this have cost on a public API"; + # treat as advisory not source-of-truth. "CleverThis-15/Qwen3-6-35B-A3B-GGUF-UD-Q3-K-XL": {"in": 0.5, "out": 1.0}, - "CleverThis/Qwen3-6-35B-A3B-GGUF-BF16": {"in": 0.5, "out": 1.0}, - "CleverThis-2/MiniMax-M2-7-GGUF-BF16": {"in": 0.5, "out": 1.0}, + "CleverThis/Qwen3-6-35B-A3B-GGUF-BF16": {"in": 0.5, "out": 1.0}, + "CleverThis-2/MiniMax-M2-7-GGUF-BF16": {"in": 0.5, "out": 1.0}, + "CleverThis-4/Kimi-K2-6-GGUF-Q2-K-XL": {"in": 0.5, "out": 1.0}, # Sentinel — when we don't know, we'd rather produce 0 than a wild # number. The UI surfaces "model unpriced" rows separately. "_unknown": {"in": 0.0, "out": 0.0}, @@ -410,17 +420,55 @@ def _load_prices() -> dict[str, dict[str, float]]: data = json.loads(override_path.read_text()) if isinstance(data, dict): merged = dict(_DEFAULT_PRICES) - merged.update(data) + # Skip metadata keys (``_comment``, ``_last_updated``, + # ``_sources``) so they don't pollute the price table. + # The ``_unknown`` sentinel is preserved because it IS + # a valid price-table entry the lookup falls back to. + for k, v in data.items(): + if k.startswith("_") and k != "_unknown": + continue + if isinstance(v, dict): + merged[k] = v return merged except (OSError, json.JSONDecodeError) as e: logger.warning("ignoring prices.json: %r", e) return dict(_DEFAULT_PRICES) -def _cost_usd(model: str, prices: dict[str, dict[str, float]], +def _price_key(provider: str | None, model: str) -> str: + """Resolve the lookup key for the price table. + + The price table is keyed by ``provider/model`` because the same + ``modelID`` can be served by multiple providers at very different + economics (e.g. ``claude-opus-4-6`` via Anthropic direct vs via a + local proxy). Falls back to the bare model name for backward + compat with manually-inserted rows that never had a provider. + """ + if provider: + return f"{provider}/{model}" + return model + + +def _cost_usd(provider: str | None, model: str, + prices: dict[str, dict[str, float]], tokens_in: int, tokens_out: int, cached: int) -> float: - p = prices.get(model) or prices["_unknown"] - in_cost = (tokens_in - cached) * p.get("in", 0.0) / 1_000_000 + """Compute USD cost for a per-model token tally. + + Lookup order: ``{provider}/{model}`` → bare ``model`` → ``_unknown``. + The fallback chain lets a price entry written without provider + namespacing still match, while still preferring the + provider-qualified entry when present. + + Token-accounting convention: ``tokens_in`` and ``cached`` are + DISJOINT populations — the scraper writes ``tokens_in`` as + OpenCode's ``info.tokens.input`` (fresh, non-cached prompt + tokens) plus any cache-write tokens; ``cached`` is + ``info.tokens.cache.read``. So total cost is the simple sum of + three line items at three rates, with no subtraction needed. + """ + p = prices.get(_price_key(provider, model)) or prices.get(model) \ + or prices.get("_unknown", {}) + in_cost = tokens_in * p.get("in", 0.0) / 1_000_000 cached_cost = cached * p.get("cached_in", p.get("in", 0.0)) / 1_000_000 out_cost = tokens_out * p.get("out", 0.0) / 1_000_000 return round(in_cost + cached_cost + out_cost, 4) @@ -1765,13 +1813,13 @@ def _api_cost(days: int) -> dict[str, Any]: } try: cur = conn.execute( - "SELECT model, COUNT(*) AS n," + "SELECT model, provider, COUNT(*) AS n," " SUM(COALESCE(tokens_in, 0)) AS in_tok," " SUM(COALESCE(tokens_out, 0)) AS out_tok," " SUM(COALESCE(cached_tokens, 0)) AS cached_tok" " FROM llm_activity" " WHERE started_at > datetime('now', ?)" - " GROUP BY model ORDER BY n DESC", + " GROUP BY model, provider ORDER BY n DESC", (f"-{int(days)} days",), ) rows = [] @@ -1781,15 +1829,18 @@ def _api_cost(days: int) -> dict[str, Any]: in_tok = int(r["in_tok"] or 0) out_tok = int(r["out_tok"] or 0) cached = int(r["cached_tok"] or 0) + model = r["model"] or "_unknown" + provider = r["provider"] usd = _cost_usd( - r["model"] or "_unknown", prices, in_tok, out_tok, cached, + provider, model, prices, in_tok, out_tok, cached, ) + key = _price_key(provider, model) rows.append( { - "model": r["model"], "n": r["n"], + "model": model, "provider": provider, "n": r["n"], "tokens_in": in_tok, "tokens_out": out_tok, "cached": cached, "usd": usd, - "priced": (r["model"] in prices), + "priced": (key in prices or model in prices), } ) total_usd += usd diff --git a/tests/auto_agents/test_telemetry_server.py b/tests/auto_agents/test_telemetry_server.py index 828ac284b..8d8bfc46c 100644 --- a/tests/auto_agents/test_telemetry_server.py +++ b/tests/auto_agents/test_telemetry_server.py @@ -384,3 +384,162 @@ def test_running_long_worker_set_when_pid_alive_and_heartbeat_stale( "the long-worker state must fire — that's the operator alarm " "for a regressed on_poll callback" ) + + +# ─── Cost calculation ───────────────────────────────────────────────── +# +# These tests pin the May 2026 fix that closed three intertwined bugs: +# (1) the lookup used bare ``model`` but ``_DEFAULT_PRICES`` was keyed +# by ``provider/model``, so even priced models showed $0; (2) the SQL +# grouped by ``model`` only, so the same modelID served by two providers +# at different rates was conflated; (3) ``tokens_in - cached`` went +# negative because the scraper records ``tokens_in`` as fresh-only, +# but the cost calc treated it as fresh + cached. + + +def _seed_llm_activity_row(cache, **overrides): + row = { + "started_at": "2026-05-01T12:00:00+00:00", + "agent": "pr-review-worker", + "model": "claude-opus-4-6", + "provider": "local-claude", + "tokens_in": 1_000_000, # 1M fresh input tokens + "tokens_out": 100_000, # 100K output tokens + "cached_tokens": 10_000_000, # 10M cached read tokens + } + row.update(overrides) + cache.upsert_llm_activity(row) + + +def test_cost_usd_uses_provider_qualified_lookup(telemetry): + """Same modelID, two providers, two different rates: the dispatch + must NOT conflate them. Regression pin for the 2026-05 bug where + the SQL grouped by ``model`` only.""" + server, _ = telemetry + # Hand-craft a price table with two distinct rates for the same model. + prices = { + "providerA/shared-model": {"in": 10.0, "out": 100.0}, + "providerB/shared-model": {"in": 1.0, "out": 10.0}, + "_unknown": {"in": 0.0, "out": 0.0}, + } + a = server._cost_usd("providerA", "shared-model", prices, 1_000_000, 1_000_000, 0) + b = server._cost_usd("providerB", "shared-model", prices, 1_000_000, 1_000_000, 0) + # A: 1M * $10/M input + 1M * $100/M output = $10 + $100 = $110 + # B: 1M * $1/M input + 1M * $10/M output = $1 + $10 = $11 + assert a == 110.0 + assert b == 11.0 + + +def test_cost_usd_falls_back_to_bare_model_then_unknown(telemetry): + """Lookup is provider/model → bare model → _unknown. Lets a + manually-inserted price entry without provider namespacing still + match, and a totally unknown model produces $0 rather than KeyError.""" + server, _ = telemetry + prices = { + "bare-model": {"in": 2.0, "out": 4.0}, + "_unknown": {"in": 0.0, "out": 0.0}, + } + # bare model fallback + assert server._cost_usd("anyprovider", "bare-model", prices, 1_000_000, 0, 0) == 2.0 + # _unknown fallback + assert server._cost_usd("anyprovider", "novel-model", prices, 1_000_000, 99, 0) == 0.0 + + +def test_cost_usd_does_not_subtract_cached_from_input(telemetry): + """``tokens_in`` and ``cached`` are DISJOINT in the schema — + they bill at different rates and must NOT be subtracted from each + other. Regression pin: a previous calc did ``(tokens_in - cached)`` + which goes negative whenever cache reads exceed fresh input + (Anthropic prompt caching makes this the common case).""" + server, _ = telemetry + prices = { + "p/m": {"in": 1.0, "out": 0.0, "cached_in": 0.1}, + "_unknown": {"in": 0.0, "out": 0.0}, + } + # 1M fresh + 10M cached: in_cost = 1M * $1/M = $1; cached_cost = 10M * $0.1/M = $1; total = $2 + usd = server._cost_usd("p", "m", prices, 1_000_000, 0, 10_000_000) + assert usd == 2.0, ( + f"expected $2.00 (fresh+cached at their own rates), got ${usd}; " + "the (tokens_in - cached) subtraction bug has regressed" + ) + + +def test_api_cost_returns_nonzero_for_priced_model(telemetry, monkeypatch): + """End-to-end pin: a seeded ``llm_activity`` row with a model + present in ``_DEFAULT_PRICES`` produces a non-zero USD total + via ``_api_cost``. This is the test that would have failed + before the lookup-key fix landed.""" + server, cache = telemetry + # Force the loader to skip any operator-edited prices.json so the + # test is deterministic against the in-code defaults. + monkeypatch.setenv("TELEMETRY_PRICES_JSON", "/nonexistent") + _seed_llm_activity_row(cache) + # Re-run started_at into a fresh window relative to "now" + cache._conn.execute( + "UPDATE llm_activity SET started_at = datetime('now', '-1 hour')" + ) + cache._conn.commit() + + payload = server._api_cost(days=7) + assert payload["instrumentation_pending"] is False + assert len(payload["rows"]) == 1 + row = payload["rows"][0] + assert row["model"] == "claude-opus-4-6" + assert row["provider"] == "local-claude" + assert row["priced"] is True + # 1M fresh * $5/M + 10M cached * $0.50/M + 100K out * $25/M + # = $5 + $5 + $2.50 = $12.50 + assert row["usd"] == 12.5, ( + f"expected $12.50, got ${row['usd']}; check _DEFAULT_PRICES " + "for local-claude/claude-opus-4-6 or the math in _cost_usd" + ) + assert payload["totals"]["usd"] == 12.5 + + +def test_api_cost_groups_by_provider_and_model(telemetry, monkeypatch): + """Two rows with the same model but different providers must + produce two separate rows in the response, NOT a single + conflated row.""" + server, cache = telemetry + monkeypatch.setenv("TELEMETRY_PRICES_JSON", "/nonexistent") + _seed_llm_activity_row( + cache, model="shared", provider="A", message_id="m1", + tokens_in=1000, tokens_out=10, cached_tokens=0, + ) + _seed_llm_activity_row( + cache, model="shared", provider="B", message_id="m2", + tokens_in=2000, tokens_out=20, cached_tokens=0, + ) + cache._conn.execute( + "UPDATE llm_activity SET started_at = datetime('now', '-1 hour')" + ) + cache._conn.commit() + + payload = server._api_cost(days=7) + providers = sorted(r["provider"] for r in payload["rows"]) + assert providers == ["A", "B"], ( + f"expected two rows for providers A and B, got {payload['rows']}" + ) + + +def test_load_prices_skips_metadata_keys(telemetry, monkeypatch, tmp_path): + """Metadata keys starting with ``_`` (e.g. ``_comment``, + ``_last_updated``, ``_sources``) must not pollute the price table + — only ``_unknown`` is preserved as a legitimate sentinel entry.""" + import json as _json + server, _ = telemetry + override = tmp_path / "prices.json" + override.write_text(_json.dumps({ + "_comment": "hello", + "_last_updated": "2026-05-17", + "_sources": {"x": "y"}, + "_unknown": {"in": 0.0, "out": 0.0}, + "test/model": {"in": 1.0, "out": 2.0}, + })) + monkeypatch.setenv("TELEMETRY_PRICES_JSON", str(override)) + loaded = server._load_prices() + for meta in ("_comment", "_last_updated", "_sources"): + assert meta not in loaded, f"metadata key {meta!r} leaked into price table" + assert "_unknown" in loaded, "_unknown sentinel must survive" + assert loaded["test/model"] == {"in": 1.0, "out": 2.0} +