cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

fix(budget): repair cost reporting BDD tests and add cost to plan status JSON

Browse Source 
- Add `cost` field to `_plan_spec_dict` in plan.py so `plan status --format
  json` includes cost metadata in output (was previously omitted)
- Fix `_make_plan_with_cost` helper: replace broken
  `Plan.namespaced_name.__class__(...)` (returns FieldInfo, not NamespacedName)
  with `NamespacedName.parse("local/test-plan")`; add `NamespacedName` import
- Remove `<plan_id>` / `<session_id>` angle-bracket placeholders from feature
  file When steps: these are not Scenario Outline templates so they were
  passed literally to `_validate_plan_ulid()` which rejected them; steps now
  use `context.plan_id` / `context.session_id` set by the Given steps
- Rename `I run plan status for the plan` step to
  `I run plan status for the plan with cost reporting` to avoid AmbiguousStep
  collision with `plan_cli_spec_alignment_steps.py`
- Fix session `estimated_cost` assertions: the value is nested under
  `token_usage` (not top-level) and is a formatted string `"$0.0080"`,
  not a float
- Apply ruff format to step file

ISSUES CLOSED: #5250
This commit is contained in:
HAL 9000
2026-06-04 10:56:08 -04:00
committed by Forgejo
parent 702c5935f2
commit 6c672c0ab2
3 changed files with 46 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
features/cost_reporting_cli.feature
+4 -4
View File
@@ -6,27 +6,27 @@ Feature: Cost reporting in plan status and session show CLI output
Scenario: Plan status JSON output includes cost field
Given a plan status CLI runner with mocked service
And a plan exists with cost metadata
When I run plan status --format json for plan <plan_id>
When I run plan status --format json for the plan
Then the JSON output contains a "cost" field
And the cost field contains cost metadata
Scenario: Session show JSON output includes estimated cost
Given a session show CLI runner with mocked service
And a session exists with token usage
When I run session show --format json for session <session_id>
When I run session show --format json for the session
Then the JSON output contains "estimated_cost" field
And the estimated_cost is properly formatted as currency
Scenario: Plan status rich output displays cost panel
Given a plan status CLI runner with mocked service
And a plan exists with cost metadata
When I run plan status for plan <plan_id>
When I run plan status for the plan with cost reporting
Then the output contains cost information
And the cost information is properly formatted
Scenario: Session show rich output displays cost information
Given a session show CLI runner with mocked service
And a session exists with token usage
When I run session show for session <session_id>
When I run session show for the session
Then the output contains cost information
And the cost information is properly formatted
features/steps/cost_reporting_cli_steps.py
+40 -57
View File
@@ -23,6 +23,7 @@ from cleveragents.cli.commands.plan import app as plan_app
from cleveragents.cli.commands.session import app as session_app
from cleveragents.domain.models.core.cost_metadata import CostMetadata
from cleveragents.domain.models.core.plan import (
NamespacedName,
Plan,
PlanIdentity,
PlanPhase,
@@ -63,9 +64,7 @@ def _make_plan_with_cost(
plan_id=plan_id or str(ULID()),
root_plan_id=plan_id or str(ULID()),
),
namespaced_name=Plan.namespaced_name.__class__(
namespace="local", name="test-plan"
),
namespaced_name=NamespacedName.parse("local/test-plan"),
description="Test plan for cost reporting",
action_name="local/test-action",
phase=PlanPhase.EXECUTE,
@@ -188,9 +187,7 @@ def step_session_with_token_usage(context: Context) -> None:
output_tokens=200,
estimated_cost=0.012,
)
session = _make_session_with_tokens(
session_id=_SESSION_ID, token_usage=token_usage
)
session = _make_session_with_tokens(session_id=_SESSION_ID, token_usage=token_usage)
context.mock_service.get.return_value = session
context.session_id = _SESSION_ID
@@ -200,9 +197,10 @@ def step_session_with_token_usage(context: Context) -> None:
# ---------------------------------------------------------------------------
@when('I run plan status --format json for plan {plan_id}')
def step_plan_status_json(context: Context, plan_id: str) -> None:
@when("I run plan status --format json for the plan")
def step_plan_status_json(context: Context) -> None:
"""Execute ``plan status <id> --format json`` and store the result."""
plan_id = context.plan_id
plan = _make_plan_with_cost(plan_id=plan_id)
context.mock_service.get_plan.return_value = plan
context.result = context.runner.invoke(
@@ -210,9 +208,10 @@ def step_plan_status_json(context: Context, plan_id: str) -> None:
)
@when('I run session show --format json for session {session_id}')
def step_session_show_json(context: Context, session_id: str) -> None:
@when("I run session show --format json for the session")
def step_session_show_json(context: Context) -> None:
"""Execute ``session show <id> --format json`` and store the result."""
session_id = context.session_id
session = _make_session_with_tokens(session_id=session_id)
context.mock_service.get.return_value = session
context.result = context.runner.invoke(
@@ -220,17 +219,19 @@ def step_session_show_json(context: Context, session_id: str) -> None:
)
@when('I run plan status for plan {plan_id}')
def step_plan_status_rich(context: Context, plan_id: str) -> None:
@when("I run plan status for the plan with cost reporting")
def step_plan_status_rich(context: Context) -> None:
"""Execute ``plan status <id>`` (rich output) and store the result."""
plan_id = context.plan_id
plan = _make_plan_with_cost(plan_id=plan_id)
context.mock_service.get_plan.return_value = plan
context.result = context.runner.invoke(plan_app, ["status", plan_id])
@when('I run session show for session {session_id}')
def step_session_show_rich(context: Context, session_id: str) -> None:
@when("I run session show for the session")
def step_session_show_rich(context: Context) -> None:
"""Execute ``session show <id>`` (rich output) and store the result."""
session_id = context.session_id
session = _make_session_with_tokens(session_id=session_id)
context.mock_service.get.return_value = session
context.result = context.runner.invoke(session_app, ["show", session_id])
@@ -245,8 +246,7 @@ def step_session_show_rich(context: Context, session_id: str) -> None:
def step_json_has_cost_field(context: Context) -> None:
"""Verify the CLI JSON output includes a ``cost`` key."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
parsed = json.loads(context.result.output)
@@ -258,65 +258,53 @@ def step_json_has_cost_field(context: Context) -> None:
def step_cost_field_has_metadata(context: Context) -> None:
"""Verify the ``cost`` field includes expected cost metadata keys."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
cost = data.get("cost", {})
assert isinstance(
cost, dict
), f"cost field should be a dict, got {type(cost).__name__}"
assert "total_cost_usd" in cost, (
"cost field should contain total_cost_usd"
)
assert "total_tokens" in cost, (
"cost field should contain total_tokens"
)
assert "input_tokens" in cost, (
"cost field should contain input_tokens"
)
assert "output_tokens" in cost, (
"cost field should contain output_tokens"
assert isinstance(cost, dict), (
f"cost field should be a dict, got {type(cost).__name__}"
)
assert "total_cost_usd" in cost, "cost field should contain total_cost_usd"
assert "total_tokens" in cost, "cost field should contain total_tokens"
assert "input_tokens" in cost, "cost field should contain input_tokens"
assert "output_tokens" in cost, "cost field should contain output_tokens"
@then('the JSON output contains "estimated_cost" field')
def step_json_has_estimated_cost(context: Context) -> None:
"""Verify JSON output includes ``estimated_cost`` for session."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
assert "estimated_cost" in data, (
f"'estimated_cost' key not found in JSON: {data}"
assert "token_usage" in data, f"'token_usage' key not found in JSON: {data}"
assert "estimated_cost" in data["token_usage"], (
f"'estimated_cost' key not found in token_usage: {data['token_usage']}"
)
@then("the estimated_cost is properly formatted as currency")
def step_estimated_cost_formatted(context: Context) -> None:
"""Verify ``estimated_cost`` is a numeric value in JSON output."""
"""Verify ``estimated_cost`` is a currency-formatted string in JSON output."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
estimated_cost = data.get("estimated_cost")
assert estimated_cost is not None, (
"estimated_cost should not be None"
token_usage = data.get("token_usage", {})
estimated_cost = token_usage.get("estimated_cost")
assert estimated_cost is not None, "estimated_cost should not be None"
assert isinstance(estimated_cost, str), (
f"estimated_cost should be a string, got {type(estimated_cost).__name__}"
)
assert isinstance(estimated_cost, (int, float)), (
f"estimated_cost should be numeric, "
f"got type={type(estimated_cost).__name__}"
)
assert estimated_cost >= 0, (
"estimated_cost should be non-negative"
assert estimated_cost.startswith("$"), (
f"estimated_cost should start with '$', got {estimated_cost!r}"
)
@@ -329,15 +317,13 @@ def step_estimated_cost_formatted(context: Context) -> None:
def step_output_has_cost_info(context: Context) -> None:
"""Verify rich text output contains cost-related keywords."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
output_lower = context.result.output.lower()
cost_keywords = ["cost", "tokens", "estimated"]
assert any(kw in output_lower for kw in cost_keywords), (
f"Output should contain cost-related keywords. Output:\n"
f"{context.result.output}"
f"Output should contain cost-related keywords. Output:\n{context.result.output}"
)
@@ -345,11 +331,8 @@ def step_output_has_cost_info(context: Context) -> None:
def step_cost_info_formatted(context: Context) -> None:
"""Verify cost information follows a reasonable format in rich output."""
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: "
f"{context.result.output}"
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# For rich output, validate that the output is non-empty
assert context.result.output.strip(), (
"Rich output should not be empty"
)
assert context.result.output.strip(), "Rich output should not be empty"
src/cleveragents/cli/commands/plan.py
+2
View File
@@ -306,6 +306,8 @@ def _plan_spec_dict(plan: Any) -> dict[str, object]:
result["last_completed_step"] = plan.last_completed_step
if plan.last_checkpoint_id:
result["last_checkpoint_id"] = plan.last_checkpoint_id
if plan.cost_metadata is not None:
result["cost"] = plan.cost_metadata.as_display_dict()
return result
# Legacy plan fallback