diff --git a/features/cost_reporting_cli.feature b/features/cost_reporting_cli.feature new file mode 100644 index 000000000..2788cb9d2 --- /dev/null +++ b/features/cost_reporting_cli.feature @@ -0,0 +1,32 @@ +Feature: Cost reporting in plan status and session show CLI output + As a user + I want to see cost information in the plan status and session show commands + So that I can track how much I've spent on plans and sessions + + Scenario: Plan status JSON output includes cost field + Given a plan status CLI runner with mocked service + And a plan exists with cost metadata + When I run plan status --format json for the plan + Then the JSON output contains a "cost" field + And the cost field contains cost metadata + + Scenario: Session show JSON output includes estimated cost + Given a session show CLI runner with mocked service + And a session exists with token usage + When I run session show --format json for the session + Then the JSON output contains "estimated_cost" field + And the estimated_cost is properly formatted as currency + + Scenario: Plan status rich output displays cost panel + Given a plan status CLI runner with mocked service + And a plan exists with cost metadata + When I run plan status for the plan with cost reporting + Then the output contains cost information + And the cost information is properly formatted + + Scenario: Session show rich output displays cost information + Given a session show CLI runner with mocked service + And a session exists with token usage + When I run session show for the session + Then the output contains cost information + And the cost information is properly formatted diff --git a/features/steps/cost_reporting_cli_steps.py b/features/steps/cost_reporting_cli_steps.py new file mode 100644 index 000000000..2999e4305 --- /dev/null +++ b/features/steps/cost_reporting_cli_steps.py @@ -0,0 +1,335 @@ +"""Step implementations for cost reporting in CLI output. + +Tests for ``agents plan status`` and ``agents session show`` commands, +verifying that cost information is correctly reported in both JSON and +rich output formats. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any +from unittest.mock import MagicMock, patch + +from behave import given, then, when +from behave.runner import Context +from typer.testing import CliRunner +from ulid import ULID + +from cleveragents.cli.commands import session as session_mod +from cleveragents.cli.commands.plan import app as plan_app +from cleveragents.cli.commands.session import app as session_app +from cleveragents.domain.models.core.cost_metadata import CostMetadata +from cleveragents.domain.models.core.plan import ( + NamespacedName, + Plan, + PlanIdentity, + PlanPhase, + ProcessingState, +) +from cleveragents.domain.models.core.session import Session, SessionTokenUsage + +# ULID constants shared across scenarios +_PLAN_ID = str(ULID()) +_SESSION_ID = str(ULID()) + +_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"} + + +def _unwrap_envelope(parsed: Any) -> Any: + """Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is.""" + if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()): + return parsed["data"] + return parsed + + +def _make_plan_with_cost( + *, + plan_id: str | None = None, + cost_metadata: CostMetadata | None = None, +) -> Plan: + """Create a test Plan with cost metadata. + + Args: + plan_id: Optional plan ULID; generates one if omitted. + cost_metadata: Optional CostMetadata; creates one with sample data if omitted. + + Returns: + A fully constructed Plan with cost tracking. + """ + return Plan( + identity=PlanIdentity( + plan_id=plan_id or str(ULID()), + root_plan_id=plan_id or str(ULID()), + ), + namespaced_name=NamespacedName.parse("local/test-plan"), + description="Test plan for cost reporting", + action_name="local/test-action", + phase=PlanPhase.EXECUTE, + processing_state=ProcessingState.PROCESSING, + cost_metadata=cost_metadata + or CostMetadata( + total_tokens=1500, + input_tokens=1000, + output_tokens=500, + total_cost=0.045, + budget_remaining=95.0, + provider_costs={"openai": 0.035}, + ), + ) + + +def _make_session_with_tokens( + *, + session_id: str | None = None, + token_usage: SessionTokenUsage | None = None, +) -> Session: + """Create a test Session with token usage / cost data. + + Args: + session_id: Optional session ULID; generates one if omitted. + token_usage: Optional SessionTokenUsage; creates one with sample data if omitted. + + Returns: + A fully constructed Session with token usage tracking. + """ + return Session( + session_id=session_id or str(ULID()), + actor_name="openai/gpt-4", + namespace="local", + messages=[], + token_usage=token_usage + or SessionTokenUsage( + input_tokens=200, + output_tokens=150, + estimated_cost=0.008, + ), + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + +# --------------------------------------------------------------------------- +# Setup +# --------------------------------------------------------------------------- + + +@given("a plan status CLI runner with mocked service") +def step_plan_cli_runner(context: Context) -> None: + """Set up CLI runner with a mocked lifecycle service for plan status.""" + context.runner = CliRunner() + context.mock_service = MagicMock() + context.plan_id = _PLAN_ID + + # Default: get_plan returns a plan with cost metadata + default_plan = _make_plan_with_cost(plan_id=_PLAN_ID) + context.mock_service.get_plan.return_value = default_plan + + # Patch the module-level service accessor, restoring original on cleanup + _patcher = patch( + "cleveragents.cli.commands.plan._get_lifecycle_service", + return_value=context.mock_service, + ) + _patcher.start() + context.add_cleanup(_patcher.stop) + + +@given("a session show CLI runner with mocked service") +def step_session_cli_runner(context: Context) -> None: + """Set up CLI runner with a mocked session service for session show.""" + context.runner = CliRunner() + context.mock_service = MagicMock() + context.session_id = _SESSION_ID + + # Default: get returns a session with token usage / cost + default_session = _make_session_with_tokens(session_id=_SESSION_ID) + context.mock_service.get.return_value = default_session + + # Patch the module-level service accessor + session_mod._service = context.mock_service # type: ignore[assignment] + + def cleanup() -> None: + session_mod._service = None # type: ignore[assignment] + + context.add_cleanup(cleanup) + + +# --------------------------------------------------------------------------- +# Given: domain objects with cost data +# --------------------------------------------------------------------------- + + +@given("a plan exists with cost metadata") +def step_plan_with_cost_metadata(context: Context) -> None: + """Create a test Plan with cost metadata and register it on the service.""" + cost_meta = CostMetadata( + total_tokens=2000, + input_tokens=1400, + output_tokens=600, + total_cost=0.06, + budget_remaining=40.0, + provider_costs={"openai": 0.04, "anthropic": 0.02}, + ) + plan = _make_plan_with_cost(plan_id=_PLAN_ID, cost_metadata=cost_meta) + context.mock_service.get_plan.return_value = plan + context.plan_id = _PLAN_ID + + +@given("a session exists with token usage") +def step_session_with_token_usage(context: Context) -> None: + """Create a test Session with token usage data and register it on the service.""" + token_usage = SessionTokenUsage( + input_tokens=300, + output_tokens=200, + estimated_cost=0.012, + ) + session = _make_session_with_tokens(session_id=_SESSION_ID, token_usage=token_usage) + context.mock_service.get.return_value = session + context.session_id = _SESSION_ID + + +# --------------------------------------------------------------------------- +# When: CLI command invocations +# --------------------------------------------------------------------------- + + +@when("I run plan status --format json for the plan") +def step_plan_status_json(context: Context) -> None: + """Execute ``plan status --format json`` and store the result.""" + plan_id = context.plan_id + plan = _make_plan_with_cost(plan_id=plan_id) + context.mock_service.get_plan.return_value = plan + context.result = context.runner.invoke( + plan_app, ["status", plan_id, "--format", "json"] + ) + + +@when("I run session show --format json for the session") +def step_session_show_json(context: Context) -> None: + """Execute ``session show --format json`` and store the result.""" + session_id = context.session_id + session = _make_session_with_tokens(session_id=session_id) + context.mock_service.get.return_value = session + context.result = context.runner.invoke( + session_app, ["show", session_id, "--format", "json"] + ) + + +@when("I run plan status for the plan with cost reporting") +def step_plan_status_rich(context: Context) -> None: + """Execute ``plan status `` (rich output) and store the result.""" + plan_id = context.plan_id + plan = _make_plan_with_cost(plan_id=plan_id) + context.mock_service.get_plan.return_value = plan + context.result = context.runner.invoke(plan_app, ["status", plan_id]) + + +@when("I run session show for the session") +def step_session_show_rich(context: Context) -> None: + """Execute ``session show `` (rich output) and store the result.""" + session_id = context.session_id + session = _make_session_with_tokens(session_id=session_id) + context.mock_service.get.return_value = session + context.result = context.runner.invoke(session_app, ["show", session_id]) + + +# --------------------------------------------------------------------------- +# JSON output assertions +# --------------------------------------------------------------------------- + + +@then('the JSON output contains a "cost" field') +def step_json_has_cost_field(context: Context) -> None: + """Verify the CLI JSON output includes a ``cost`` key.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) + assert "cost" in data, f"'cost' key not found in JSON: {data}" + + +@then("the cost field contains cost metadata") +def step_cost_field_has_metadata(context: Context) -> None: + """Verify the ``cost`` field includes expected cost metadata keys.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) + cost = data.get("cost", {}) + assert isinstance(cost, dict), ( + f"cost field should be a dict, got {type(cost).__name__}" + ) + assert "total_cost_usd" in cost, "cost field should contain total_cost_usd" + assert "total_tokens" in cost, "cost field should contain total_tokens" + assert "input_tokens" in cost, "cost field should contain input_tokens" + assert "output_tokens" in cost, "cost field should contain output_tokens" + + +@then('the JSON output contains "estimated_cost" field') +def step_json_has_estimated_cost(context: Context) -> None: + """Verify JSON output includes ``estimated_cost`` for session.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) + assert "token_usage" in data, f"'token_usage' key not found in JSON: {data}" + assert "estimated_cost" in data["token_usage"], ( + f"'estimated_cost' key not found in token_usage: {data['token_usage']}" + ) + + +@then("the estimated_cost is properly formatted as currency") +def step_estimated_cost_formatted(context: Context) -> None: + """Verify ``estimated_cost`` is a currency-formatted string in JSON output.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + parsed = json.loads(context.result.output) + data = _unwrap_envelope(parsed) + token_usage = data.get("token_usage", {}) + estimated_cost = token_usage.get("estimated_cost") + assert estimated_cost is not None, "estimated_cost should not be None" + assert isinstance(estimated_cost, str), ( + f"estimated_cost should be a string, got {type(estimated_cost).__name__}" + ) + assert estimated_cost.startswith("$"), ( + f"estimated_cost should start with '$', got {estimated_cost!r}" + ) + + +# --------------------------------------------------------------------------- +# Rich output assertions +# --------------------------------------------------------------------------- + + +@then("the output contains cost information") +def step_output_has_cost_info(context: Context) -> None: + """Verify rich text output contains cost-related keywords.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + output_lower = context.result.output.lower() + cost_keywords = ["cost", "tokens", "estimated"] + assert any(kw in output_lower for kw in cost_keywords), ( + f"Output should contain cost-related keywords. Output:\n{context.result.output}" + ) + + +@then("the cost information is properly formatted") +def step_cost_info_formatted(context: Context) -> None: + """Verify cost information follows a reasonable format in rich output.""" + assert context.result.exit_code == 0, ( + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" + ) + + # For rich output, validate that the output is non-empty + assert context.result.output.strip(), "Rich output should not be empty" diff --git a/src/cleveragents/cli/commands/plan.py b/src/cleveragents/cli/commands/plan.py index 13d0e3116..f05f73870 100644 --- a/src/cleveragents/cli/commands/plan.py +++ b/src/cleveragents/cli/commands/plan.py @@ -306,6 +306,8 @@ def _plan_spec_dict(plan: Any) -> dict[str, object]: result["last_completed_step"] = plan.last_completed_step if plan.last_checkpoint_id: result["last_checkpoint_id"] = plan.last_checkpoint_id + if plan.cost_metadata is not None: + result["cost"] = plan.cost_metadata.as_display_dict() return result # Legacy plan fallback