diff --git a/features/cost_reporting_cli.feature b/features/cost_reporting_cli.feature index b2c6e170e..2788cb9d2 100644 --- a/features/cost_reporting_cli.feature +++ b/features/cost_reporting_cli.feature @@ -6,27 +6,27 @@ Feature: Cost reporting in plan status and session show CLI output Scenario: Plan status JSON output includes cost field Given a plan status CLI runner with mocked service And a plan exists with cost metadata - When I run plan status --format json for plan + When I run plan status --format json for the plan Then the JSON output contains a "cost" field And the cost field contains cost metadata Scenario: Session show JSON output includes estimated cost Given a session show CLI runner with mocked service And a session exists with token usage - When I run session show --format json for session + When I run session show --format json for the session Then the JSON output contains "estimated_cost" field And the estimated_cost is properly formatted as currency Scenario: Plan status rich output displays cost panel Given a plan status CLI runner with mocked service And a plan exists with cost metadata - When I run plan status for plan + When I run plan status for the plan with cost reporting Then the output contains cost information And the cost information is properly formatted Scenario: Session show rich output displays cost information Given a session show CLI runner with mocked service And a session exists with token usage - When I run session show for session + When I run session show for the session Then the output contains cost information And the cost information is properly formatted diff --git a/features/steps/cost_reporting_cli_steps.py b/features/steps/cost_reporting_cli_steps.py index e34e41abf..cd1140c7f 100644 --- a/features/steps/cost_reporting_cli_steps.py +++ b/features/steps/cost_reporting_cli_steps.py @@ -23,6 +23,7 @@ from cleveragents.cli.commands.plan import app as plan_app from cleveragents.cli.commands.session import app as session_app from cleveragents.domain.models.core.cost_metadata import CostMetadata from cleveragents.domain.models.core.plan import ( + NamespacedName, Plan, PlanIdentity, PlanPhase, @@ -63,9 +64,7 @@ def _make_plan_with_cost( plan_id=plan_id or str(ULID()), root_plan_id=plan_id or str(ULID()), ), - namespaced_name=Plan.namespaced_name.__class__( - namespace="local", name="test-plan" - ), + namespaced_name=NamespacedName.parse("local/test-plan"), description="Test plan for cost reporting", action_name="local/test-action", phase=PlanPhase.EXECUTE, @@ -188,9 +187,7 @@ def step_session_with_token_usage(context: Context) -> None: output_tokens=200, estimated_cost=0.012, ) - session = _make_session_with_tokens( - session_id=_SESSION_ID, token_usage=token_usage - ) + session = _make_session_with_tokens(session_id=_SESSION_ID, token_usage=token_usage) context.mock_service.get.return_value = session context.session_id = _SESSION_ID @@ -200,9 +197,10 @@ def step_session_with_token_usage(context: Context) -> None: # --------------------------------------------------------------------------- -@when('I run plan status --format json for plan {plan_id}') -def step_plan_status_json(context: Context, plan_id: str) -> None: +@when("I run plan status --format json for the plan") +def step_plan_status_json(context: Context) -> None: """Execute ``plan status --format json`` and store the result.""" + plan_id = context.plan_id plan = _make_plan_with_cost(plan_id=plan_id) context.mock_service.get_plan.return_value = plan context.result = context.runner.invoke( @@ -210,9 +208,10 @@ def step_plan_status_json(context: Context, plan_id: str) -> None: ) -@when('I run session show --format json for session {session_id}') -def step_session_show_json(context: Context, session_id: str) -> None: +@when("I run session show --format json for the session") +def step_session_show_json(context: Context) -> None: """Execute ``session show --format json`` and store the result.""" + session_id = context.session_id session = _make_session_with_tokens(session_id=session_id) context.mock_service.get.return_value = session context.result = context.runner.invoke( @@ -220,17 +219,19 @@ def step_session_show_json(context: Context, session_id: str) -> None: ) -@when('I run plan status for plan {plan_id}') -def step_plan_status_rich(context: Context, plan_id: str) -> None: +@when("I run plan status for the plan with cost reporting") +def step_plan_status_rich(context: Context) -> None: """Execute ``plan status `` (rich output) and store the result.""" + plan_id = context.plan_id plan = _make_plan_with_cost(plan_id=plan_id) context.mock_service.get_plan.return_value = plan context.result = context.runner.invoke(plan_app, ["status", plan_id]) -@when('I run session show for session {session_id}') -def step_session_show_rich(context: Context, session_id: str) -> None: +@when("I run session show for the session") +def step_session_show_rich(context: Context) -> None: """Execute ``session show `` (rich output) and store the result.""" + session_id = context.session_id session = _make_session_with_tokens(session_id=session_id) context.mock_service.get.return_value = session context.result = context.runner.invoke(session_app, ["show", session_id]) @@ -245,8 +246,7 @@ def step_session_show_rich(context: Context, session_id: str) -> None: def step_json_has_cost_field(context: Context) -> None: """Verify the CLI JSON output includes a ``cost`` key.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) parsed = json.loads(context.result.output) @@ -258,65 +258,53 @@ def step_json_has_cost_field(context: Context) -> None: def step_cost_field_has_metadata(context: Context) -> None: """Verify the ``cost`` field includes expected cost metadata keys.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) parsed = json.loads(context.result.output) data = _unwrap_envelope(parsed) cost = data.get("cost", {}) - assert isinstance( - cost, dict - ), f"cost field should be a dict, got {type(cost).__name__}" - assert "total_cost_usd" in cost, ( - "cost field should contain total_cost_usd" - ) - assert "total_tokens" in cost, ( - "cost field should contain total_tokens" - ) - assert "input_tokens" in cost, ( - "cost field should contain input_tokens" - ) - assert "output_tokens" in cost, ( - "cost field should contain output_tokens" + assert isinstance(cost, dict), ( + f"cost field should be a dict, got {type(cost).__name__}" ) + assert "total_cost_usd" in cost, "cost field should contain total_cost_usd" + assert "total_tokens" in cost, "cost field should contain total_tokens" + assert "input_tokens" in cost, "cost field should contain input_tokens" + assert "output_tokens" in cost, "cost field should contain output_tokens" @then('the JSON output contains "estimated_cost" field') def step_json_has_estimated_cost(context: Context) -> None: """Verify JSON output includes ``estimated_cost`` for session.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) parsed = json.loads(context.result.output) data = _unwrap_envelope(parsed) - assert "estimated_cost" in data, ( - f"'estimated_cost' key not found in JSON: {data}" + assert "token_usage" in data, f"'token_usage' key not found in JSON: {data}" + assert "estimated_cost" in data["token_usage"], ( + f"'estimated_cost' key not found in token_usage: {data['token_usage']}" ) @then("the estimated_cost is properly formatted as currency") def step_estimated_cost_formatted(context: Context) -> None: - """Verify ``estimated_cost`` is a numeric value in JSON output.""" + """Verify ``estimated_cost`` is a currency-formatted string in JSON output.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) parsed = json.loads(context.result.output) data = _unwrap_envelope(parsed) - estimated_cost = data.get("estimated_cost") - assert estimated_cost is not None, ( - "estimated_cost should not be None" + token_usage = data.get("token_usage", {}) + estimated_cost = token_usage.get("estimated_cost") + assert estimated_cost is not None, "estimated_cost should not be None" + assert isinstance(estimated_cost, str), ( + f"estimated_cost should be a string, got {type(estimated_cost).__name__}" ) - assert isinstance(estimated_cost, (int, float)), ( - f"estimated_cost should be numeric, " - f"got type={type(estimated_cost).__name__}" - ) - assert estimated_cost >= 0, ( - "estimated_cost should be non-negative" + assert estimated_cost.startswith("$"), ( + f"estimated_cost should start with '$', got {estimated_cost!r}" ) @@ -329,15 +317,13 @@ def step_estimated_cost_formatted(context: Context) -> None: def step_output_has_cost_info(context: Context) -> None: """Verify rich text output contains cost-related keywords.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) output_lower = context.result.output.lower() cost_keywords = ["cost", "tokens", "estimated"] assert any(kw in output_lower for kw in cost_keywords), ( - f"Output should contain cost-related keywords. Output:\n" - f"{context.result.output}" + f"Output should contain cost-related keywords. Output:\n{context.result.output}" ) @@ -345,11 +331,8 @@ def step_output_has_cost_info(context: Context) -> None: def step_cost_info_formatted(context: Context) -> None: """Verify cost information follows a reasonable format in rich output.""" assert context.result.exit_code == 0, ( - f"Expected exit code 0, got {context.result.exit_code}: " - f"{context.result.output}" + f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}" ) # For rich output, validate that the output is non-empty - assert context.result.output.strip(), ( - "Rich output should not be empty" - ) + assert context.result.output.strip(), "Rich output should not be empty" diff --git a/src/cleveragents/cli/commands/plan.py b/src/cleveragents/cli/commands/plan.py index 13d0e3116..f05f73870 100644 --- a/src/cleveragents/cli/commands/plan.py +++ b/src/cleveragents/cli/commands/plan.py @@ -306,6 +306,8 @@ def _plan_spec_dict(plan: Any) -> dict[str, object]: result["last_completed_step"] = plan.last_completed_step if plan.last_checkpoint_id: result["last_checkpoint_id"] = plan.last_checkpoint_id + if plan.cost_metadata is not None: + result["cost"] = plan.cost_metadata.as_display_dict() return result # Legacy plan fallback