Compare commits

...

2 Commits

Author SHA1 Message Date
HAL9000 0b13d3cbb5 fix(governance): resolve lint and test issues in cost tracking implementation 2026-04-21 08:21:29 +00:00
HAL9000 ad6b4fd7ce feat(governance): implement cost and session budget tracking and enforcement
CI / lint (pull_request) Failing after 51s
CI / push-validation (pull_request) Successful in 38s
CI / helm (pull_request) Successful in 39s
CI / unit_tests (pull_request) Failing after 2m43s
CI / quality (pull_request) Successful in 4m16s
CI / typecheck (pull_request) Successful in 4m33s
CI / security (pull_request) Successful in 4m43s
CI / docker (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / build (pull_request) Successful in 3m49s
CI / e2e_tests (pull_request) Successful in 7m6s
CI / integration_tests (pull_request) Successful in 7m59s
CI / status-check (pull_request) Failing after 3s
- Add BudgetExceededError exception for budget limit violations
- Implement CostTrackingService for managing cost tracking and budget enforcement
- Add comprehensive BDD tests for cost budget tracking and enforcement
- Support three-tier budget hierarchy (plan, session, organization)
- Track token usage and costs per provider
- Enforce budget limits with warning thresholds at 90% utilization
- Record budget exhaustion events for auditability
2026-04-18 17:52:04 +00:00
4 changed files with 615 additions and 0 deletions
@@ -0,0 +1,79 @@
Feature: Cost and Session Budget Tracking and Enforcement
Background:
Given a cost tracking service is initialized
And a session cost budget of $10.00 is configured
And a cost metadata tracker is created
Scenario: Track cost usage after LLM call
When an LLM call costs $2.50 with 100 input tokens and 50 output tokens
Then the cost metadata should show:
| total_cost | 2.50 |
| total_tokens | 150 |
| input_tokens | 100 |
| output_tokens | 50 |
And the session budget should show $7.50 remaining
Scenario: Enforce budget limit when exceeded
Given the session budget is $5.00
When an LLM call costs $3.00
And another LLM call costs $2.50
Then the second call should raise BudgetExceededError
And the error should indicate plan budget exceeded
Scenario: Track provider costs separately
When an LLM call from OpenAI costs $1.50
And an LLM call from Anthropic costs $2.00
Then the cost metadata should show provider breakdown:
| openai | 1.50 |
| anthropic | 2.00 |
Scenario: Query budget status
Given the session budget is $10.00
When $3.50 has been spent
Then budget status should show:
| limit | 10.00 |
| used | 3.50 |
| remaining | 6.50 |
| utilization | 0.35 |
Scenario: Warn at 90% budget utilization
Given the session budget is $10.00
When $9.00 has been spent
And a check for $0.50 cost is performed
Then the budget check should return warning=true
And the check should still be allowed
Scenario: Record budget exhaustion events
Given the session budget is $5.00
When an LLM call costs $5.50 from OpenAI using gpt-4o
Then a BudgetExceededError should be raised
And the cost metadata should record an exhaustion event with:
| budget_type | plan |
| limit | 5.00 |
| used | 5.50 |
| provider | openai |
| model | gpt-4o |
Scenario: Handle unlimited budgets
Given no session budget is configured
When multiple LLM calls totaling $100.00 are made
Then all calls should succeed
And the cost metadata should track the total cost
Scenario: Three-tier budget hierarchy
Given a session budget of $50.00
And an organization budget of $100.00
When $40.00 has been spent at session level
And $80.00 has been spent at organization level
And a check for $15.00 cost is performed
Then the check should fail due to organization budget
And the exceeded_level should be "org"
Scenario: Budget check with negative cost
When a budget check is performed with a negative cost
Then a ValueError should be raised
Scenario: Record cost with invalid tokens
When recording cost with negative input tokens
Then a ValueError should be raised
@@ -0,0 +1,231 @@
"""Step definitions for cost budget tracking and enforcement."""
from __future__ import annotations
from behave import given, when, then
from cleveragents.core.cost_tracking import CostTrackingService
from cleveragents.domain.models.core.cost_budget import (
BudgetCheckResult,
BudgetLevel,
SessionCostBudget,
ThreadSafeOrgCostAccumulator,
)
from cleveragents.domain.models.core.cost_metadata import CostMetadata
@given("a cost tracking service")
def step_create_cost_tracking_service(context):
"""Create a cost tracking service."""
context.cost_tracking_service = CostTrackingService()
@given("a session budget of ${max_cost}")
def step_create_session_budget(context, max_cost):
"""Create a session budget."""
context.session_budget = SessionCostBudget(max_cost_usd=float(max_cost))
@given("an organization cost accumulator with limit ${max_cost}")
def step_create_org_accumulator(context, max_cost):
"""Create an organization cost accumulator."""
context.org_accumulator = ThreadSafeOrgCostAccumulator(
max_cost_usd=float(max_cost)
)
@given("cost metadata")
def step_create_cost_metadata(context):
"""Create cost metadata."""
context.cost_metadata = CostMetadata()
@when("an LLM call costs ${cost} with {input_tokens} input tokens and {output_tokens} output tokens")
def step_llm_call_with_tokens(context, cost, input_tokens, output_tokens):
"""Record an LLM call with specific token counts."""
context.proposed_cost = float(cost)
context.input_tokens = int(input_tokens)
context.output_tokens = int(output_tokens)
@when("an LLM call costs ${cost} from {provider} with model {model}")
def step_llm_call_with_provider_model(context, cost, provider, model):
"""Record an LLM call with provider and model information."""
context.proposed_cost = float(cost)
context.provider = provider
context.model = model
context.input_tokens = 100
context.output_tokens = 50
@when("I check if the cost is allowed")
def step_check_budget(context):
"""Check if the proposed cost is allowed."""
session_budget = getattr(context, 'session_budget', None)
org_accumulator = getattr(context, 'org_accumulator', None)
proposed_cost = getattr(context, 'proposed_cost', 0.0)
context.budget_check_result = context.cost_tracking_service.check_budget(
session_budget=session_budget,
org_accumulator=org_accumulator,
proposed_cost=proposed_cost,
)
@when("I record the cost")
def step_record_cost(context):
"""Record the cost in the tracking service."""
session_budget = getattr(context, 'session_budget', None)
org_accumulator = getattr(context, 'org_accumulator', None)
cost_metadata = getattr(context, 'cost_metadata', CostMetadata())
proposed_cost = getattr(context, 'proposed_cost', 0.0)
input_tokens = getattr(context, 'input_tokens', 100)
output_tokens = getattr(context, 'output_tokens', 50)
context.cost_tracking_service.record_cost(
cost_metadata=cost_metadata,
session_budget=session_budget,
org_accumulator=org_accumulator,
input_tokens=input_tokens,
output_tokens=output_tokens,
cost=proposed_cost,
provider="openai",
model="gpt-4",
)
@when("I enforce the budget")
def step_enforce_budget(context):
"""Enforce the budget limits."""
session_budget = getattr(context, 'session_budget', None)
org_accumulator = getattr(context, 'org_accumulator', None)
cost_metadata = getattr(context, 'cost_metadata', CostMetadata())
try:
context.cost_tracking_service.enforce_budget(
session_budget=session_budget,
org_accumulator=org_accumulator,
cost_metadata=cost_metadata,
provider="openai",
model="gpt-4",
)
context.budget_exceeded = False
except Exception as e:
context.budget_exceeded = True
context.budget_error = str(e)
@then("the cost should be allowed")
def step_cost_allowed(context):
"""Verify that the cost is allowed."""
assert context.budget_check_result.allowed, (
f"Cost should be allowed but got: {context.budget_check_result.reason}"
)
@then("the cost should not be allowed")
def step_cost_not_allowed(context):
"""Verify that the cost is not allowed."""
assert not context.budget_check_result.allowed, (
"Cost should not be allowed"
)
@then("the exceeded level should be {level}")
def step_check_exceeded_level(context, level):
"""Verify the exceeded budget level."""
expected_level = BudgetLevel[level.upper()]
assert context.budget_check_result.exceeded_level == expected_level, (
f"Expected {expected_level} but got {context.budget_check_result.exceeded_level}"
)
@then("a warning should be issued")
def step_warning_issued(context):
"""Verify that a warning is issued."""
assert context.budget_check_result.warning, (
"Warning should be issued"
)
@then("no warning should be issued")
def step_no_warning(context):
"""Verify that no warning is issued."""
assert not context.budget_check_result.warning, (
"No warning should be issued"
)
@then("the session budget should have recorded the cost")
def step_session_budget_recorded(context):
"""Verify that the session budget recorded the cost."""
session_budget = getattr(context, 'session_budget', None)
if session_budget:
assert session_budget.total_cost > 0, (
"Session budget should have recorded the cost"
)
@then("the organization budget should have recorded the cost")
def step_org_budget_recorded(context):
"""Verify that the organization budget recorded the cost."""
org_accumulator = getattr(context, 'org_accumulator', None)
if org_accumulator:
assert org_accumulator.total_cost > 0, (
"Organization budget should have recorded the cost"
)
@then("the budget should be exceeded")
def step_budget_exceeded(context):
"""Verify that the budget is exceeded."""
assert context.budget_exceeded, (
"Budget should be exceeded"
)
@then("the budget should not be exceeded")
def step_budget_not_exceeded(context):
"""Verify that the budget is not exceeded."""
assert not context.budget_exceeded, (
f"Budget should not be exceeded: {getattr(context, 'budget_error', '')}"
)
@given("the session budget utilization is at {utilization}%")
def step_set_session_utilization(context, utilization):
"""Set the session budget utilization."""
session_budget = getattr(context, 'session_budget', None)
if session_budget:
# Calculate the cost needed to reach the desired utilization
target_cost = session_budget.max_cost_usd * (float(utilization) / 100)
session_budget.record_cost(target_cost)
@given("the organization budget utilization is at {utilization}%")
def step_set_org_utilization(context, utilization):
"""Set the organization budget utilization."""
org_accumulator = getattr(context, 'org_accumulator', None)
if org_accumulator:
# Calculate the cost needed to reach the desired utilization
target_cost = org_accumulator.max_cost_usd * (float(utilization) / 100)
org_accumulator.record_cost(target_cost)
@then("the budget status should show session limit of ${limit}")
def step_check_session_limit(context, limit):
"""Verify the session budget limit."""
session_budget = getattr(context, 'session_budget', None)
if session_budget:
assert session_budget.max_cost_usd == float(limit), (
f"Expected limit {limit} but got {session_budget.max_cost_usd}"
)
@then("the budget status should show organization limit of ${limit}")
def step_check_org_limit(context, limit):
"""Verify the organization budget limit."""
org_accumulator = getattr(context, 'org_accumulator', None)
if org_accumulator:
assert org_accumulator.max_cost_usd == float(limit), (
f"Expected limit {limit} but got {org_accumulator.max_cost_usd}"
)
+258
View File
@@ -0,0 +1,258 @@
"""Cost tracking and budget enforcement service.
Implements cost tracking, budget enforcement, and provider fallback controls
for AI provider usage. Tracks token usage, monetary costs, and enforces
configurable budget limits at plan and session levels.
"""
from __future__ import annotations
import threading
from typing import Any
from cleveragents.core.exceptions import BudgetExceededError
from cleveragents.domain.models.core.cost_budget import (
BudgetCheckResult,
BudgetLevel,
SessionCostBudget,
ThreadSafeOrgCostAccumulator,
)
from cleveragents.domain.models.core.cost_metadata import (
BudgetExhaustionEvent,
CostMetadata,
)
class CostTrackingService:
"""Service for tracking costs and enforcing budget limits.
Manages cost tracking at plan, session, and organization levels.
Enforces budget limits and raises BudgetExceededError when limits
are exceeded.
Thread safety:
- Session budgets are protected by an external lock (caller's responsibility)
- Org accumulators use ThreadSafeOrgCostAccumulator for thread safety
"""
def __init__(self) -> None:
"""Initialize the cost tracking service."""
self._lock = threading.Lock()
def check_budget(
self,
session_budget: SessionCostBudget | None,
org_accumulator: ThreadSafeOrgCostAccumulator | None,
proposed_cost: float,
) -> BudgetCheckResult:
"""Check if a proposed cost fits within all budget tiers.
Args:
session_budget: Session-level budget (None = unlimited).
org_accumulator: Organization-level accumulator (None = unlimited).
proposed_cost: Cost to check in USD.
Returns:
BudgetCheckResult indicating whether the cost is allowed.
Raises:
ValueError: If proposed_cost is negative.
"""
if proposed_cost < 0:
raise ValueError("proposed_cost must be non-negative")
# Check session budget
if session_budget is not None and session_budget.would_exceed(
proposed_cost
):
utilization = session_budget.utilization()
warning = (
utilization is not None and utilization >= 0.9
if not session_budget.is_exceeded()
else False
)
return BudgetCheckResult(
allowed=False,
exceeded_level=BudgetLevel.SESSION,
reason=(
f"Session budget of ${session_budget.max_cost_usd:.2f} "
f"would be exceeded (current: ${session_budget.total_cost:.2f}, "
f"proposed: ${proposed_cost:.2f})"
),
warning=warning,
)
# Check org budget
if org_accumulator is not None and org_accumulator.would_exceed(
proposed_cost
):
utilization = org_accumulator.utilization()
warning = (
utilization is not None and utilization >= 0.9
if not org_accumulator.is_exceeded()
else False
)
return BudgetCheckResult(
allowed=False,
exceeded_level=BudgetLevel.ORG,
reason=(
f"Organization budget of ${org_accumulator.max_cost_usd:.2f} "
f"would be exceeded (current: ${org_accumulator.total_cost:.2f}, "
f"proposed: ${proposed_cost:.2f})"
),
warning=warning,
)
# Check for warning thresholds
warning = False
if session_budget is not None:
utilization = session_budget.utilization()
if utilization is not None and utilization >= 0.9:
warning = True
if org_accumulator is not None:
utilization = org_accumulator.utilization()
if utilization is not None and utilization >= 0.9:
warning = True
return BudgetCheckResult(allowed=True, warning=warning)
def record_cost(
self,
cost_metadata: CostMetadata,
session_budget: SessionCostBudget | None,
org_accumulator: ThreadSafeOrgCostAccumulator | None,
*,
input_tokens: int,
output_tokens: int,
cost: float,
provider: str,
model: str = "",
) -> None:
"""Record cost usage and update budgets.
Args:
cost_metadata: Cost metadata to update.
session_budget: Session budget to update (None = skip).
org_accumulator: Org accumulator to update (None = skip).
input_tokens: Input tokens consumed.
output_tokens: Output tokens consumed.
cost: Cost in USD.
provider: Provider name.
model: Model name (optional).
Raises:
ValueError: If any numeric argument is invalid.
"""
# Record in cost metadata
cost_metadata.record_usage(
input_tokens=input_tokens,
output_tokens=output_tokens,
cost=cost,
provider=provider,
)
# Update session budget
if session_budget is not None:
session_budget.record_cost(cost)
cost_metadata.budget_remaining = session_budget.remaining()
# Update org accumulator
if org_accumulator is not None:
org_accumulator.record_cost(cost)
def enforce_budget(
self,
session_budget: SessionCostBudget | None,
org_accumulator: ThreadSafeOrgCostAccumulator | None,
cost_metadata: CostMetadata,
*,
provider: str,
model: str = "",
) -> None:
"""Enforce budget limits and raise error if exceeded.
Args:
session_budget: Session budget to check.
org_accumulator: Org accumulator to check.
cost_metadata: Cost metadata for recording exhaustion events.
provider: Provider name for error reporting.
model: Model name for error reporting.
Raises:
BudgetExceededError: If any budget limit is exceeded.
"""
# Check session budget
if session_budget is not None and session_budget.is_exceeded():
event = BudgetExhaustionEvent(
budget_type="plan",
limit=session_budget.max_cost_usd or 0.0,
used=session_budget.total_cost,
provider=provider,
model=model,
)
cost_metadata.budget_exhaustion_events.append(event)
raise BudgetExceededError(
budget_type="plan",
limit=session_budget.max_cost_usd or 0.0,
used=session_budget.total_cost,
provider=provider,
model=model,
)
# Check org budget
if org_accumulator is not None and org_accumulator.is_exceeded():
event = BudgetExhaustionEvent(
budget_type="daily",
limit=org_accumulator.max_cost_usd or 0.0,
used=org_accumulator.total_cost,
provider=provider,
model=model,
)
cost_metadata.budget_exhaustion_events.append(event)
raise BudgetExceededError(
budget_type="daily",
limit=org_accumulator.max_cost_usd or 0.0,
used=org_accumulator.total_cost,
provider=provider,
model=model,
)
def get_budget_status(
self,
session_budget: SessionCostBudget | None,
org_accumulator: ThreadSafeOrgCostAccumulator | None,
) -> dict[str, Any]:
"""Get current budget status.
Args:
session_budget: Session budget to query.
org_accumulator: Org accumulator to query.
Returns:
Dictionary with budget status information.
"""
status: dict[str, Any] = {}
if session_budget is not None:
status["session"] = {
"limit": session_budget.max_cost_usd,
"used": session_budget.total_cost,
"remaining": session_budget.remaining(),
"utilization": session_budget.utilization(),
}
if org_accumulator is not None:
status["organization"] = {
"limit": org_accumulator.max_cost_usd,
"used": org_accumulator.total_cost,
"remaining": org_accumulator.remaining(),
"utilization": org_accumulator.utilization(),
}
return status
__all__ = [
"CostTrackingService",
]
+47
View File
@@ -355,3 +355,50 @@ __all__ = [
"ToolTypeMismatchError",
"ValidationError",
]
class BudgetExceededError(DomainError):
"""Raised when a cost budget limit is exceeded.
Attributes:
budget_type: The type of budget that was exceeded ('plan' or 'daily').
limit: The budget limit in USD.
used: The amount used in USD.
provider: The provider that triggered the exhaustion.
model: The model that triggered the exhaustion.
"""
def __init__(
self,
budget_type: str,
limit: float,
used: float,
provider: str = "",
model: str = "",
details: dict[str, Any] | None = None,
) -> None:
"""Initialize budget exceeded error.
Args:
budget_type: Type of budget ('plan' or 'daily').
limit: Budget limit in USD.
used: Amount used in USD.
provider: Provider that triggered exhaustion.
model: Model that triggered exhaustion.
details: Additional error context.
"""
message = (
f"{budget_type.capitalize()} budget of ${limit:.2f} exceeded "
f"(used: ${used:.2f})"
)
if provider:
message += f" by {provider}"
if model:
message += f" ({model})"
super().__init__(message, details)
self.budget_type = budget_type
self.limit = limit
self.used = used
self.provider = provider
self.model = model