feat(provider): add cost controls and fallback #428

Merged
freemo merged 1 commits from feature/m4-provider-costs into master 2026-02-25 17:40:20 +00:00
15 changed files with 3034 additions and 0 deletions
+6
View File
@@ -2,6 +2,12 @@
## Unreleased
- Added token/cost tracking, budget enforcement (per-plan and per-day), provider fallback
selection with capability filtering, and cost metadata for plan execution. New config keys
`budget_per_plan`, `budget_per_day`, and `fallback_providers` control spending limits and
provider ordering. Budget warnings are emitted at 90% usage, and requests are blocked at 100%.
Per-provider cost table includes default token cost estimates for offline reporting. Budget
exhaustion events are persisted in plan metadata for auditability. (#324)
- Added plan-level and project-level advisory locking with configurable timeouts, re-entrant
acquisition, conflict detection, lock renewal, graceful shutdown release, startup cleanup of
expired locks, and diagnostics check for stale lock reporting. (#327)
+144
View File
@@ -0,0 +1,144 @@
"""ASV benchmarks for cost controls overhead.
Measures performance of:
- CostEntry creation and estimation
- ProviderCostTable lookups
- CostTracker budget checks and recording
- FallbackSelector provider selection
"""
from __future__ import annotations
import importlib
import sys
from pathlib import Path
# Ensure the local source tree is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
import cleveragents # noqa: E402
importlib.reload(cleveragents)
from cleveragents.config.settings import Settings # noqa: E402
from cleveragents.domain.models.core.cost_metadata import CostMetadata # noqa: E402
from cleveragents.providers.cost_table import ( # noqa: E402
CostEntry,
ProviderCostTable,
)
from cleveragents.providers.cost_tracker import CostTracker # noqa: E402
from cleveragents.providers.fallback_selector import FallbackSelector # noqa: E402
from cleveragents.providers.registry import ProviderRegistry # noqa: E402
class TimeCostEntry:
"""Benchmark CostEntry creation and estimation."""
def time_create_cost_entry(self) -> None:
CostEntry(input_cost_per_token=0.00001, output_cost_per_token=0.00003)
def time_estimate_cost(self) -> None:
entry = CostEntry(input_cost_per_token=0.00001, output_cost_per_token=0.00003)
entry.estimate_cost(1000, 500)
class TimeCostTable:
"""Benchmark ProviderCostTable lookups."""
def setup(self) -> None:
self.table = ProviderCostTable()
def time_lookup_known_model(self) -> None:
self.table.get_cost_entry("openai", "gpt-4o")
def time_lookup_unknown_model(self) -> None:
self.table.get_cost_entry("openai", "unknown-model")
def time_list_providers(self) -> None:
self.table.list_providers()
def time_list_models(self) -> None:
self.table.list_models("openai")
class TimeCostTracker:
"""Benchmark CostTracker budget checks and recording."""
def setup(self) -> None:
self.tracker = CostTracker(budget_per_plan=10.0, budget_per_day=50.0)
self.tracker_unlimited = CostTracker()
def time_record_usage_with_budget(self) -> None:
metadata = CostMetadata()
self.tracker.record_usage(
metadata,
provider="openai",
model="gpt-4o",
input_tokens=1000,
output_tokens=500,
)
def time_record_usage_unlimited(self) -> None:
metadata = CostMetadata()
self.tracker_unlimited.record_usage(
metadata,
provider="openai",
model="gpt-4o",
input_tokens=1000,
output_tokens=500,
)
def time_check_plan_budget(self) -> None:
metadata = CostMetadata()
self.tracker.check_plan_budget(metadata)
def time_check_daily_budget(self) -> None:
self.tracker.check_daily_budget()
def time_estimate_cost(self) -> None:
self.tracker.estimate_cost("openai", "gpt-4o", 1000, 500)
class TimeFallbackSelector:
"""Benchmark FallbackSelector provider selection."""
def setup(self) -> None:
settings = Settings()
self.registry = ProviderRegistry(settings)
self.selector = FallbackSelector(registry=self.registry)
def time_select_no_requirements(self) -> None:
self.selector.select()
def time_select_with_tool_calls(self) -> None:
self.selector.select(require_tool_calls=True)
def time_select_with_all_requirements(self) -> None:
self.selector.select(
require_tool_calls=True,
require_streaming=True,
require_vision=True,
require_json_mode=True,
)
class TimeCostMetadata:
"""Benchmark CostMetadata operations."""
def time_create_metadata(self) -> None:
CostMetadata()
def time_record_usage(self) -> None:
meta = CostMetadata()
meta.record_usage(
input_tokens=1000,
output_tokens=500,
cost=0.025,
provider="openai",
)
def time_display_dict(self) -> None:
meta = CostMetadata(total_tokens=1500, total_cost=0.025)
meta.as_display_dict()
+111
View File
@@ -0,0 +1,111 @@
# Cost Controls
CleverAgents provides cost tracking, budget enforcement, and provider
fallback controls for AI provider usage.
## Configuration Keys
| Key | Env Variable | Type | Default | Description |
|-----|-------------|------|---------|-------------|
| `budget_per_plan` | `CLEVERAGENTS_BUDGET_PER_PLAN` | `float \| None` | `None` | Max USD spend per plan execution. `None` = unlimited. |
| `budget_per_day` | `CLEVERAGENTS_BUDGET_PER_DAY` | `float \| None` | `None` | Max USD spend per calendar day. `None` = unlimited. |
| `fallback_providers` | `CLEVERAGENTS_FALLBACK_PROVIDERS` | `list[str]` | `[]` | Ordered provider fallback list. Empty uses default order. |
## Budget Thresholds
| Threshold | Value | Behavior |
|-----------|-------|----------|
| Warning | 90% of limit | Log warning, continue execution |
| Block | 100% of limit | Block further provider calls, persist exhaustion event |
## Default Fallback Order
When `fallback_providers` is empty, the default order is:
1. OpenAI
2. Anthropic
3. Google
4. Azure
5. OpenRouter
6. Groq
7. Together
8. Cohere
Providers are skipped if they:
- Are not configured (no API key)
- Lack required capabilities (tool calling, streaming, vision, JSON mode)
- Would exceed the daily budget
## Cost Tracking Fields
Plan execution metadata includes:
| Field | Type | Description |
|-------|------|-------------|
| `total_tokens` | `int` | Total tokens consumed (input + output) |
| `input_tokens` | `int` | Input/prompt tokens |
| `output_tokens` | `int` | Output/completion tokens |
| `total_cost` | `float` | Estimated cost in USD |
| `budget_remaining` | `float \| None` | Remaining plan budget (None if unlimited) |
| `provider_costs` | `dict[str, float]` | Per-provider cost breakdown |
| `budget_exhaustion_events` | `list` | Audit log of budget limit events |
## Viewing Costs
Use `agents plan status` to see cost data for a plan:
```
agents plan status <plan_id>
```
Cost data appears under the `cost` key in the output.
## Per-Provider Cost Table
Default token cost estimates are provided for offline reporting.
Models not in the table use a conservative default estimate.
### Supported Providers
- **OpenAI**: gpt-4o, gpt-4o-mini, gpt-4-turbo
- **Anthropic**: claude-sonnet-4-20250514, claude-3-5-haiku-20241022, claude-opus-4-20250514
- **Google**: gemini-2.0-flash, gemini-1.5-pro
- **Groq**: llama-3.1-70b-versatile
- **Together**: meta-llama/Llama-3.1-70B-Instruct-Turbo
- **Cohere**: command-r-plus
- **Mock**: mock-gpt (zero cost)
## Budget Exhaustion Events
When a budget limit is reached, an exhaustion event is persisted to the
plan's cost metadata with:
- Timestamp
- Budget type (`plan` or `daily`)
- Limit amount
- Used amount
- Provider and model that triggered it
These events are retained for auditability and appear in `plan status`
output.
## Examples
### Set a per-plan budget
```bash
export CLEVERAGENTS_BUDGET_PER_PLAN=5.00
```
### Set a daily budget
```bash
export CLEVERAGENTS_BUDGET_PER_DAY=50.00
```
### Configure fallback providers
```bash
export CLEVERAGENTS_FALLBACK_PROVIDERS='["anthropic", "openai", "groq"]'
```
+463
View File
@@ -0,0 +1,463 @@
Feature: Cost controls and provider fallback
As a developer using CleverAgents
I want cost tracking, budget enforcement, and provider fallback
So that I can control AI spending and ensure service continuity
# ---- CostEntry ----
@unit @cost
Scenario: CostEntry validates non-negative input cost
Given I import the CostEntry class
When I create a CostEntry with negative input cost
Then a ValueError should be raised for CostEntry
@unit @cost
Scenario: CostEntry validates non-negative output cost
Given I import the CostEntry class
When I create a CostEntry with negative output cost
Then a ValueError should be raised for CostEntry
@unit @cost
Scenario: CostEntry estimates cost correctly
Given I import the CostEntry class
When I create a CostEntry with input cost 0.00001 and output cost 0.00003
And I estimate cost for 1000 input tokens and 500 output tokens
Then the estimated cost should be 0.025
@unit @cost
Scenario: CostEntry rejects negative token counts
Given I import the CostEntry class
When I create a CostEntry with input cost 0.00001 and output cost 0.00003
Then estimating cost with negative input tokens raises ValueError
And estimating cost with negative output tokens raises ValueError
# ---- ProviderCostTable ----
@unit @cost
Scenario: ProviderCostTable returns default cost for unknown model
Given I create a ProviderCostTable with defaults
When I look up cost for provider "openai" model "unknown-model-xyz"
Then the cost entry should be the default cost
@unit @cost
Scenario: ProviderCostTable returns known cost for gpt-4o
Given I create a ProviderCostTable with defaults
When I look up cost for provider "openai" model "gpt-4o"
Then the cost entry input cost should be 0.0000025
And the cost entry output cost should be 0.00001
@unit @cost
Scenario: ProviderCostTable with custom entries overrides defaults
Given I create a ProviderCostTable with custom entry for "openai" "gpt-4o"
When I look up cost for provider "openai" model "gpt-4o"
Then the cost entry input cost should be 0.001
And the cost entry output cost should be 0.002
@unit @cost
Scenario: ProviderCostTable validates empty provider
Given I create a ProviderCostTable with defaults
Then looking up cost for empty provider raises ValueError
@unit @cost
Scenario: ProviderCostTable validates empty model
Given I create a ProviderCostTable with defaults
Then looking up cost for empty model raises ValueError
@unit @cost
Scenario: ProviderCostTable lists providers and models
Given I create a ProviderCostTable with defaults
When I list providers from the cost table
Then the provider list should include "openai"
And the provider list should include "anthropic"
When I list models for provider "openai"
Then the model list should include "gpt-4o"
@unit @cost
Scenario: ProviderCostTable rejects non-dict custom entries
When I create a ProviderCostTable with non-dict custom entries
Then a TypeError should be raised for ProviderCostTable
# ---- CostMetadata ----
@unit @cost
Scenario: CostMetadata records usage correctly
Given I create a fresh CostMetadata
When I record usage of 100 input tokens and 50 output tokens at cost 0.01 for provider "openai"
Then total tokens should be 150
And total cost should be 0.01
And provider costs for "openai" should be 0.01
@unit @cost
Scenario: CostMetadata accumulates multiple usages
Given I create a fresh CostMetadata
When I record usage of 100 input tokens and 50 output tokens at cost 0.01 for provider "openai"
And I record usage of 200 input tokens and 100 output tokens at cost 0.02 for provider "anthropic"
Then total tokens should be 450
And total cost should be 0.03
And provider costs for "openai" should be 0.01
And provider costs for "anthropic" should be 0.02
@unit @cost
Scenario: CostMetadata rejects negative input tokens
Given I create a fresh CostMetadata
Then recording usage with negative input tokens raises ValueError
@unit @cost
Scenario: CostMetadata rejects negative cost
Given I create a fresh CostMetadata
Then recording usage with negative cost raises ValueError
@unit @cost
Scenario: CostMetadata rejects empty provider
Given I create a fresh CostMetadata
Then recording usage with empty provider raises ValueError
@unit @cost
Scenario: CostMetadata display dict includes all fields
Given I create a fresh CostMetadata
When I record usage of 100 input tokens and 50 output tokens at cost 0.01 for provider "openai"
And I get the display dict from cost metadata
Then the display dict should have key "total_tokens"
And the display dict should have key "total_cost_usd"
@unit @cost
Scenario: CostMetadata display dict includes budget remaining when set
Given I create a fresh CostMetadata
When I set budget remaining to 5.0
And I get the display dict from cost metadata
Then the display dict should have key "budget_remaining_usd"
# ---- BudgetExhaustionEvent ----
@unit @cost
Scenario: BudgetExhaustionEvent validates budget type
Given I import the BudgetExhaustionEvent class
When I create a BudgetExhaustionEvent with invalid budget type
Then a validation error should be raised for BudgetExhaustionEvent
@unit @cost
Scenario: BudgetExhaustionEvent accepts plan budget type
Given I import the BudgetExhaustionEvent class
When I create a BudgetExhaustionEvent with budget type "plan"
Then the event should be created successfully
@unit @cost
Scenario: BudgetExhaustionEvent accepts daily budget type
Given I import the BudgetExhaustionEvent class
When I create a BudgetExhaustionEvent with budget type "daily"
Then the event should be created successfully
# ---- CostTracker ----
@unit @cost
Scenario: CostTracker with no budget limits
Given I create a CostTracker with no limits
When I record tracked usage of 1000 input and 500 output for "openai" "gpt-4o"
Then the budget check should be under budget
And the cost metadata total tokens should be 1500
@unit @cost
Scenario: CostTracker warns at 90% of plan budget
Given I create a CostTracker with plan budget 0.001
And I create a fresh CostMetadata
When I record tracked usage of 500 input and 500 output for "openai" "gpt-4o"
Then the budget check should be warning or exceeded
@unit @cost
Scenario: CostTracker blocks at 100% of plan budget
Given I create a CostTracker with plan budget 0.0000001
And I create a fresh CostMetadata
When I record tracked usage of 10000 input and 5000 output for "openai" "gpt-4o"
Then the budget check should be exceeded
@unit @cost
Scenario: CostTracker blocks at 100% of daily budget
Given I create a CostTracker with daily budget 0.0000001
And I create a fresh CostMetadata
When I record tracked usage of 10000 input and 5000 output for "openai" "gpt-4o"
Then the budget check should be exceeded
@unit @cost
Scenario: CostTracker persists exhaustion events
Given I create a CostTracker with plan budget 0.0000001
And I create a fresh CostMetadata
When I record tracked usage of 10000 input and 5000 output for "openai" "gpt-4o"
Then the cost metadata should have budget exhaustion events
@unit @cost
Scenario: CostTracker validates negative plan budget
When I create a CostTracker with negative plan budget
Then a ValueError should be raised for CostTracker
@unit @cost
Scenario: CostTracker validates negative daily budget
When I create a CostTracker with negative daily budget
Then a ValueError should be raised for CostTracker
@unit @cost
Scenario: CostTracker estimate_cost delegates to cost table
Given I create a CostTracker with no limits
When I estimate cost for "openai" "gpt-4o" with 1000 input and 500 output
Then the estimated tracker cost should be positive
@unit @cost
Scenario: CostTracker validates empty provider in estimate
Given I create a CostTracker with no limits
Then estimating cost with empty provider raises ValueError on tracker
@unit @cost
Scenario: CostTracker validates empty model in estimate
Given I create a CostTracker with no limits
Then estimating cost with empty model raises ValueError on tracker
@unit @cost
Scenario: CostTracker validates negative tokens in record
Given I create a CostTracker with no limits
And I create a fresh CostMetadata
Then recording tracked usage with negative input raises ValueError
@unit @cost
Scenario: CostTracker check_plan_budget with no limit returns under budget
Given I create a CostTracker with no limits
And I create a fresh CostMetadata
When I check plan budget
Then the plan budget check should be under budget
@unit @cost
Scenario: CostTracker check_daily_budget with no limit returns under budget
Given I create a CostTracker with no limits
When I check daily budget
Then the daily budget check should be under budget
@unit @cost
Scenario: CostTracker get_daily_spend returns zero initially
Given I create a CostTracker with no limits
Then the daily spend should be 0.0
@unit @cost
Scenario: CostTracker budget properties are accessible
Given I create a CostTracker with plan budget 10.0 and daily budget 50.0
Then the tracker plan budget should be 10.0
And the tracker daily budget should be 50.0
@unit @cost
Scenario: CostTracker get_cost_entry validates empty provider
Given I create a CostTracker with no limits
Then getting cost entry with empty provider raises ValueError
@unit @cost
Scenario: CostTracker get_cost_entry validates empty model
Given I create a CostTracker with no limits
Then getting cost entry with empty model raises ValueError
@unit @cost
Scenario: CostTracker validates empty provider in record_usage
Given I create a CostTracker with no limits
And I create a fresh CostMetadata
Then recording tracked usage with empty provider raises ValueError
@unit @cost
Scenario: CostTracker validates empty model in record_usage
Given I create a CostTracker with no limits
And I create a fresh CostMetadata
Then recording tracked usage with empty model raises ValueError
@unit @cost
Scenario: CostTracker with zero plan budget returns exceeded
Given I create a CostTracker with plan budget 0.0
And I create a fresh CostMetadata
When I check plan budget
Then the plan budget check should be exceeded
@unit @cost
Scenario: CostTracker with zero daily budget returns exceeded
Given I create a CostTracker with daily budget 0.0
When I check daily budget
Then the daily budget check should be exceeded
# ---- FallbackSelector ----
@unit @cost
Scenario: FallbackSelector with no configured providers returns None
Given I create a ProviderRegistry with no API keys for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider
Then the fallback result provider should be None
And the fallback result should have skipped entries
@unit @cost
Scenario: FallbackSelector skips providers missing tool calls
Given I create a ProviderRegistry with mock provider for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider requiring tool calls
Then the fallback result should skip mock for missing tool calls
@unit @cost
Scenario: FallbackSelector selects first capable provider
Given I create a ProviderRegistry with openai key for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider requiring tool calls
Then the fallback result provider type should be "openai"
@unit @cost
Scenario: FallbackSelector uses custom fallback order
Given I create a ProviderRegistry with openai key for fallback
When I create a FallbackSelector with custom order "openai"
And I select a fallback provider
Then the fallback result provider type should be "openai"
@unit @cost
Scenario: FallbackSelector rejects None registry
When I create a FallbackSelector with None registry
Then a TypeError should be raised for FallbackSelector
@unit @cost
Scenario: FallbackSelector rejects empty string in fallback list
Given I create a ProviderRegistry with no API keys for fallback
When I create a FallbackSelector with empty string in fallback list
Then a ValueError should be raised for FallbackSelector
@unit @cost
Scenario: FallbackSelector skips provider when daily budget exceeded
Given I create a ProviderRegistry with openai key for fallback
And I create a CostTracker with daily budget 0.0
When I create a FallbackSelector with cost tracker
And I select a fallback provider
Then the fallback result provider should be None
# ---- Settings config keys ----
@unit @cost
Scenario: Settings budget_per_plan defaults to None
When I load cost control settings with defaults
Then budget_per_plan should be None
@unit @cost
Scenario: Settings budget_per_day defaults to None
When I load cost control settings with defaults
Then budget_per_day should be None
@unit @cost
Scenario: Settings fallback_providers defaults to empty list
When I load cost control settings with defaults
Then fallback_providers should be an empty list
@unit @cost
Scenario: Settings budget_per_plan accepts positive value
When I load cost control settings with budget_per_plan 10.0
Then budget_per_plan should be 10.0
@unit @cost
Scenario: Settings budget_per_day accepts positive value
When I load cost control settings with budget_per_day 50.0
Then budget_per_day should be 50.0
# ---- Plan model cost_metadata ----
@unit @cost
Scenario: Plan model includes cost_metadata field
Given I create a minimal Plan with cost metadata
Then the plan should have cost_metadata set
@unit @cost
Scenario: Plan as_cli_dict includes cost when present
Given I create a minimal Plan with cost metadata
When I get the CLI dict from the plan
Then the CLI dict should have a "cost" key
@unit @cost
Scenario: Plan as_cli_dict omits cost when None
Given I create a minimal Plan without cost metadata
When I get the CLI dict from the plan
Then the CLI dict should not have a "cost" key
@unit @cost
Scenario: CostMetadata display dict with exhaustion events
Given I create a fresh CostMetadata
When I add a budget exhaustion event of type "plan"
And I get the display dict from cost metadata
Then the display dict should have key "budget_exhaustion_events"
@unit @cost
Scenario: CostMetadata records negative output tokens rejection
Given I create a fresh CostMetadata
Then recording usage with negative output tokens raises ValueError
# ---- Additional coverage scenarios ----
@unit @cost
Scenario: ProviderCostTable custom entries for brand new provider
Given I create a ProviderCostTable with custom entry for new provider "myprovider" model "mymodel"
When I look up cost for provider "myprovider" model "mymodel"
Then the cost entry input cost should be 0.005
And the cost entry output cost should be 0.01
@unit @cost
Scenario: ProviderCostTable list_models with empty provider raises ValueError
Given I create a ProviderCostTable with defaults
Then listing models for empty provider raises ValueError
@unit @cost
Scenario: CostTracker estimate_cost rejects negative input tokens
Given I create a CostTracker with no limits
Then estimating cost with negative input tokens raises ValueError on tracker
@unit @cost
Scenario: CostTracker estimate_cost rejects negative output tokens
Given I create a CostTracker with no limits
Then estimating cost with negative output tokens raises ValueError on tracker
@unit @cost
Scenario: CostTracker record_usage rejects negative output tokens
Given I create a CostTracker with no limits
And I create a fresh CostMetadata
Then recording tracked usage with negative output raises ValueError
@unit @cost
Scenario: CostTracker plan budget warning at 90 percent
Given I create a CostTracker with plan budget 0.10
And I create a fresh CostMetadata
When I record tracked usage of 3600 input and 8500 output for "openai" "gpt-4o"
Then the budget check should be warning
@unit @cost
Scenario: CostTracker daily budget warning at 90 percent
Given I create a CostTracker with daily budget 0.10
And I create a fresh CostMetadata
When I record tracked usage of 3600 input and 8500 output for "openai" "gpt-4o"
Then the budget check should be warning
@unit @cost
Scenario: CostTracker get_cost_entry returns valid entry
Given I create a CostTracker with no limits
When I get cost entry for "openai" "gpt-4o"
Then the returned cost entry should have positive input cost
@unit @cost
Scenario: FallbackSelector skips unknown provider types
Given I create a ProviderRegistry with no API keys for fallback
When I create a FallbackSelector with custom order "nonexistent_provider"
And I select a fallback provider
Then the fallback result provider should be None
And the fallback skipped should mention "unknown provider type"
@unit @cost
Scenario: FallbackSelector skips providers missing streaming
Given I create a ProviderRegistry with mock provider for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider requiring streaming
Then the fallback result should skip for missing streaming
@unit @cost
Scenario: FallbackSelector skips providers missing vision
Given I create a ProviderRegistry with mock provider for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider requiring vision
Then the fallback result should skip for missing vision
@unit @cost
Scenario: FallbackSelector skips providers missing json mode
Given I create a ProviderRegistry with mock provider for fallback
When I create a FallbackSelector with the registry
And I select a fallback provider requiring json mode
Then the fallback result should skip for missing json mode
File diff suppressed because it is too large Load Diff
+71
View File
@@ -0,0 +1,71 @@
*** Settings ***
Resource ${CURDIR}/common.resource
Library OperatingSystem
Library Process
*** Variables ***
${PYTHON} python
${SRC_DIR} ${CURDIR}/..
*** Test Cases ***
Cost Entry Creation And Estimation
[Documentation] Verify CostEntry creation and cost estimation
[Tags] cost_controls provider model
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py cost-entry
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cost-entry-ok
Provider Cost Table Lookup
[Documentation] Verify ProviderCostTable model lookup
[Tags] cost_controls provider model
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py cost-table
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cost-table-ok
Cost Tracker Budget Enforcement
[Documentation] Verify CostTracker budget enforcement
[Tags] cost_controls provider budget
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py cost-tracker
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cost-tracker-budget-ok
Fallback Selector No Providers
[Documentation] Verify FallbackSelector with no configured providers
[Tags] cost_controls provider fallback
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py fallback-selector
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} fallback-selector-ok
Settings Cost Control Defaults
[Documentation] Verify Settings cost control config defaults
[Tags] cost_controls config settings
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py settings-defaults
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} settings-defaults-ok
Cost Metadata Display Dict
[Documentation] Verify CostMetadata display dictionary
[Tags] cost_controls provider model
${result}= Run Process ${PYTHON} robot/helper_cost_controls.py cost-metadata
... cwd=${SRC_DIR}
Log Process Failure ${result}
Should Be Equal As Integers ${result.rc} 0
Should Contain ${result.stdout} cost-metadata-display-ok
*** Keywords ***
Log Process Failure
[Arguments] ${result}
Run Keyword If ${result.rc} == 0 Return From Keyword
Log To Console Process failed with rc=${result.rc}
Log To Console STDOUT:${\n}${result.stdout}
Log To Console STDERR:${\n}${result.stderr}
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""Helper script for cost controls Robot Framework integration tests."""
from __future__ import annotations
import sys
from pathlib import Path
# Ensure local source is importable
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
def test_cost_entry_creation() -> None:
"""Verify CostEntry creation and estimation."""
from cleveragents.providers.cost_table import CostEntry
entry = CostEntry(input_cost_per_token=0.00001, output_cost_per_token=0.00003)
cost = entry.estimate_cost(1000, 500)
assert cost > 0.0, f"Expected positive cost, got {cost}"
print("cost-entry-ok")
def test_cost_table_lookup() -> None:
"""Verify ProviderCostTable lookup."""
from cleveragents.providers.cost_table import ProviderCostTable
table = ProviderCostTable()
entry = table.get_cost_entry("openai", "gpt-4o")
assert entry.input_cost_per_token > 0
providers = table.list_providers()
assert "openai" in providers
print("cost-table-ok")
def test_cost_tracker_budget() -> None:
"""Verify CostTracker budget enforcement."""
from cleveragents.domain.models.core.cost_metadata import CostMetadata
from cleveragents.providers.cost_tracker import BudgetStatus, CostTracker
tracker = CostTracker(budget_per_plan=0.0001)
metadata = CostMetadata()
result = tracker.record_usage(
metadata,
provider="openai",
model="gpt-4o",
input_tokens=10000,
output_tokens=5000,
)
# With a tiny budget, usage should exceed it
assert result.status in (BudgetStatus.WARNING, BudgetStatus.EXCEEDED), (
f"Expected warning/exceeded, got {result.status}"
)
print("cost-tracker-budget-ok")
def test_fallback_selector() -> None:
"""Verify FallbackSelector with no configured providers."""
import os
from cleveragents.config.settings import Settings
from cleveragents.providers.fallback_selector import FallbackSelector
from cleveragents.providers.registry import ProviderRegistry
# Clear all API keys so no providers are configured
api_key_vars = [
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"GOOGLE_API_KEY",
"AZURE_OPENAI_API_KEY",
"AZURE_API_KEY",
"OPENROUTER_API_KEY",
"GEMINI_API_KEY",
"GOOGLE_GEMINI_API_KEY",
"GROQ_API_KEY",
"TOGETHER_API_KEY",
"COHERE_API_KEY",
"PERPLEXITY_API_KEY",
]
saved: dict[str, str] = {}
for var in api_key_vars:
val = os.environ.pop(var, None)
if val is not None:
saved[var] = val
try:
settings = Settings()
registry = ProviderRegistry(settings)
selector = FallbackSelector(registry=registry)
result = selector.select()
# With no API keys, no provider should be selected
assert result.provider_type is None
assert len(result.skipped) > 0
print("fallback-selector-ok")
finally:
# Restore any keys we removed
os.environ.update(saved)
def test_settings_defaults() -> None:
"""Verify Settings cost control defaults."""
from cleveragents.config.settings import Settings
settings = Settings()
assert settings.budget_per_plan is None
assert settings.budget_per_day is None
assert settings.fallback_providers == []
print("settings-defaults-ok")
def test_cost_metadata_display() -> None:
"""Verify CostMetadata display dict."""
from cleveragents.domain.models.core.cost_metadata import CostMetadata
meta = CostMetadata()
meta.record_usage(
input_tokens=100,
output_tokens=50,
cost=0.01,
provider="openai",
)
display = meta.as_display_dict()
assert "total_tokens" in display
assert "total_cost_usd" in display
print("cost-metadata-display-ok")
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: helper_cost_controls.py <test-name>", file=sys.stderr)
sys.exit(1)
test_name = sys.argv[1]
tests = {
"cost-entry": test_cost_entry_creation,
"cost-table": test_cost_table_lookup,
"cost-tracker": test_cost_tracker_budget,
"fallback-selector": test_fallback_selector,
"settings-defaults": test_settings_defaults,
"cost-metadata": test_cost_metadata_display,
}
test_fn = tests.get(test_name)
if test_fn is None:
print(f"Unknown test: {test_name}", file=sys.stderr)
sys.exit(1)
test_fn()
+28
View File
@@ -187,6 +187,34 @@ class Settings(BaseSettings):
),
)
# Cost controls (M4)
budget_per_plan: float | None = Field(
default=None,
ge=0.0,
validation_alias=AliasChoices("CLEVERAGENTS_BUDGET_PER_PLAN"),
description=(
"Maximum USD spend per plan execution. None (default) means unlimited."
),
)
budget_per_day: float | None = Field(
default=None,
ge=0.0,
validation_alias=AliasChoices("CLEVERAGENTS_BUDGET_PER_DAY"),
description=(
"Maximum USD spend per calendar day across all plans. "
"None (default) means unlimited."
),
)
fallback_providers: list[str] = Field(
default_factory=list,
validation_alias=AliasChoices("CLEVERAGENTS_FALLBACK_PROVIDERS"),
description=(
"Ordered list of provider names to try when the primary "
"provider is unavailable or over budget. "
"Empty list uses the default fallback order."
),
)
# Persistence
database_url: str = Field(
default="sqlite:///cleveragents.db",
@@ -0,0 +1,162 @@
"""Cost metadata model for plan execution tracking.
Tracks token usage, monetary costs, and budget exhaustion events
attached to plan execution lifecycle.
"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from pydantic import BaseModel, ConfigDict, Field, field_validator
class BudgetExhaustionEvent(BaseModel):
"""Record of a budget limit being reached during plan execution.
Persisted in plan metadata for auditability (issue #324).
"""
timestamp: datetime = Field(
default_factory=datetime.now,
description="When the budget threshold was reached",
)
budget_type: str = Field(
...,
min_length=1,
description="Type of budget exhausted: 'plan' or 'daily'",
)
limit: float = Field(
...,
ge=0.0,
description="The budget limit that was reached (USD)",
)
used: float = Field(
...,
ge=0.0,
description="Amount used at time of exhaustion (USD)",
)
provider: str = Field(
default="",
description="Provider that triggered the exhaustion",
)
model: str = Field(
default="",
description="Model that triggered the exhaustion",
)
@field_validator("budget_type")
@classmethod
def validate_budget_type(cls: type[BudgetExhaustionEvent], v: str) -> str:
allowed = {"plan", "daily"}
if v not in allowed:
raise ValueError(f"budget_type must be one of {allowed}, got {v!r}")
return v
model_config = ConfigDict(validate_assignment=True)
class CostMetadata(BaseModel):
"""Cost and token tracking metadata for a plan execution.
Attached to plan execution to provide cost visibility and
auditability. Surfaced in ``plan status`` CLI output.
"""
total_tokens: int = Field(
default=0,
ge=0,
description="Total tokens consumed (input + output)",
)
input_tokens: int = Field(
default=0,
ge=0,
description="Input/prompt tokens consumed",
)
output_tokens: int = Field(
default=0,
ge=0,
description="Output/completion tokens consumed",
)
total_cost: float = Field(
default=0.0,
ge=0.0,
description="Total estimated cost in USD",
)
budget_remaining: float | None = Field(
default=None,
description="Remaining budget in USD (None if unlimited)",
)
provider_costs: dict[str, float] = Field(
default_factory=dict,
description="Cost breakdown by provider name",
)
budget_exhaustion_events: list[BudgetExhaustionEvent] = Field(
default_factory=list,
description="Audit log of budget limit events",
)
def record_usage(
self,
*,
input_tokens: int,
output_tokens: int,
cost: float,
provider: str,
) -> None:
"""Record token usage and cost from a provider call.
Args:
input_tokens: Number of input tokens consumed.
output_tokens: Number of output tokens consumed.
cost: Estimated cost in USD.
provider: Provider name for cost breakdown.
Raises:
ValueError: If any numeric argument is negative.
"""
if input_tokens < 0:
raise ValueError(f"input_tokens must be >= 0, got {input_tokens}")
if output_tokens < 0:
raise ValueError(f"output_tokens must be >= 0, got {output_tokens}")
if cost < 0.0:
raise ValueError(f"cost must be >= 0.0, got {cost}")
if not provider:
raise ValueError("provider must be a non-empty string")
self.input_tokens += input_tokens
self.output_tokens += output_tokens
self.total_tokens += input_tokens + output_tokens
self.total_cost += cost
self.provider_costs[provider] = self.provider_costs.get(provider, 0.0) + cost
def as_display_dict(self) -> dict[str, Any]:
"""Return a dictionary suitable for CLI display."""
result: dict[str, Any] = {
"total_tokens": self.total_tokens,
"input_tokens": self.input_tokens,
"output_tokens": self.output_tokens,
"total_cost_usd": round(self.total_cost, 6),
}
if self.budget_remaining is not None:
result["budget_remaining_usd"] = round(self.budget_remaining, 6)
if self.provider_costs:
result["provider_costs"] = {
k: round(v, 6) for k, v in self.provider_costs.items()
}
if self.budget_exhaustion_events:
result["budget_exhaustion_events"] = [
{
"timestamp": evt.timestamp.isoformat(),
"budget_type": evt.budget_type,
"limit": evt.limit,
"used": evt.used,
"provider": evt.provider,
"model": evt.model,
}
for evt in self.budget_exhaustion_events
]
return result
model_config = ConfigDict(validate_assignment=True)
@@ -63,6 +63,8 @@ from typing import Any, ClassVar
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from cleveragents.domain.models.core.cost_metadata import CostMetadata
# ULID is 26 characters, all uppercase alphanumeric (Crockford's base32)
ULID_PATTERN = r"^[0-9A-HJKMNP-TV-Z]{26}$"
@@ -640,6 +642,12 @@ class Plan(BaseModel):
description="ULID of the root decision node in the decision tree",
)
# Cost tracking (M4)
cost_metadata: CostMetadata | None = Field(
default=None,
description="Token/cost tracking and budget exhaustion events",
)
# Metadata
created_by: str | None = Field(None, description="User/session that created plan")
tags: list[str] = Field(default_factory=list, description="Tags for organization")
@@ -861,6 +869,8 @@ class Plan(BaseModel):
result["parent_plan_id"] = self.identity.parent_plan_id
if self.has_subplans:
result["subplan_count"] = len(self.subplan_statuses)
if self.cost_metadata is not None:
result["cost"] = self.cost_metadata.as_display_dict()
return result
model_config = ConfigDict(
+10
View File
@@ -1,3 +1,6 @@
from .cost_table import CostEntry, ProviderCostTable
from .cost_tracker import BudgetCheckResult, BudgetStatus, CostTracker
from .fallback_selector import FallbackResult, FallbackSelector
from .registry import (
ProviderRegistry,
get_provider_registry,
@@ -5,6 +8,13 @@ from .registry import (
)
__all__ = [
"BudgetCheckResult",
"BudgetStatus",
"CostEntry",
"CostTracker",
"FallbackResult",
"FallbackSelector",
"ProviderCostTable",
"ProviderRegistry",
"get_provider_registry",
"reset_provider_registry",
+216
View File
@@ -0,0 +1,216 @@
"""Per-provider cost table with default token cost estimates.
Provides offline cost estimation for token usage across supported
providers and models. Costs are in USD per token.
Default estimates are approximations for common models and should be
updated as provider pricing changes.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import ClassVar
@dataclass(frozen=True, slots=True)
class CostEntry:
"""Cost per token for a specific provider/model combination.
Attributes:
input_cost_per_token: USD cost per input/prompt token.
output_cost_per_token: USD cost per output/completion token.
"""
input_cost_per_token: float
output_cost_per_token: float
def __post_init__(self) -> None:
if self.input_cost_per_token < 0.0:
raise ValueError(
f"input_cost_per_token must be >= 0, got {self.input_cost_per_token}"
)
if self.output_cost_per_token < 0.0:
raise ValueError(
f"output_cost_per_token must be >= 0, got {self.output_cost_per_token}"
)
def estimate_cost(self, input_tokens: int, output_tokens: int) -> float:
"""Estimate the cost for a given token count.
Args:
input_tokens: Number of input tokens.
output_tokens: Number of output tokens.
Returns:
Estimated cost in USD.
Raises:
ValueError: If token counts are negative.
"""
if input_tokens < 0:
raise ValueError(f"input_tokens must be >= 0, got {input_tokens}")
if output_tokens < 0:
raise ValueError(f"output_tokens must be >= 0, got {output_tokens}")
return (
self.input_cost_per_token * input_tokens
+ self.output_cost_per_token * output_tokens
)
class ProviderCostTable:
"""Lookup table for provider/model cost estimates.
Maintains a registry of known cost entries and provides a fallback
default for unknown models. Thread-safe for read operations.
Example::
table = ProviderCostTable()
entry = table.get_cost_entry("openai", "gpt-4o")
cost = entry.estimate_cost(input_tokens=1000, output_tokens=500)
"""
# Default fallback cost for unknown models (conservative estimate)
DEFAULT_COST: ClassVar[CostEntry] = CostEntry(
input_cost_per_token=0.000005,
output_cost_per_token=0.000015,
)
# Default cost entries for known provider/model combinations.
# Costs are approximate and based on published pricing as of 2025.
_DEFAULT_ENTRIES: ClassVar[dict[str, dict[str, CostEntry]]] = {
"openai": {
"gpt-4o": CostEntry(
input_cost_per_token=0.0000025,
output_cost_per_token=0.00001,
),
"gpt-4o-mini": CostEntry(
input_cost_per_token=0.00000015,
output_cost_per_token=0.0000006,
),
"gpt-4-turbo": CostEntry(
input_cost_per_token=0.00001,
output_cost_per_token=0.00003,
),
},
"anthropic": {
"claude-sonnet-4-20250514": CostEntry(
input_cost_per_token=0.000003,
output_cost_per_token=0.000015,
),
"claude-3-5-haiku-20241022": CostEntry(
input_cost_per_token=0.0000008,
output_cost_per_token=0.000004,
),
"claude-opus-4-20250514": CostEntry(
input_cost_per_token=0.000015,
output_cost_per_token=0.000075,
),
},
"google": {
"gemini-2.0-flash": CostEntry(
input_cost_per_token=0.0000001,
output_cost_per_token=0.0000004,
),
"gemini-1.5-pro": CostEntry(
input_cost_per_token=0.00000125,
output_cost_per_token=0.000005,
),
},
"groq": {
"llama-3.1-70b-versatile": CostEntry(
input_cost_per_token=0.00000059,
output_cost_per_token=0.00000079,
),
},
"together": {
"meta-llama/Llama-3.1-70B-Instruct-Turbo": CostEntry(
input_cost_per_token=0.00000088,
output_cost_per_token=0.00000088,
),
},
"cohere": {
"command-r-plus": CostEntry(
input_cost_per_token=0.0000025,
output_cost_per_token=0.00001,
),
},
"mock": {
"mock-gpt": CostEntry(
input_cost_per_token=0.0,
output_cost_per_token=0.0,
),
},
}
def __init__(
self,
custom_entries: dict[str, dict[str, CostEntry]] | None = None,
) -> None:
"""Initialize the cost table.
Args:
custom_entries: Optional custom cost entries to merge with
defaults. Custom entries take precedence.
Raises:
TypeError: If custom_entries is not a dict.
"""
if custom_entries is not None and not isinstance(custom_entries, dict):
raise TypeError(
f"custom_entries must be a dict, got {type(custom_entries)}"
)
self._entries: dict[str, dict[str, CostEntry]] = {}
for provider, models in self._DEFAULT_ENTRIES.items():
self._entries[provider] = dict(models)
if custom_entries:
for provider, models in custom_entries.items():
if provider not in self._entries:
self._entries[provider] = {}
self._entries[provider].update(models)
def get_cost_entry(self, provider: str, model: str) -> CostEntry:
"""Look up the cost entry for a provider/model pair.
Args:
provider: Provider name (e.g. 'openai', 'anthropic').
model: Model identifier (e.g. 'gpt-4o').
Returns:
The matching CostEntry, or DEFAULT_COST if not found.
Raises:
ValueError: If provider or model is empty.
"""
if not provider:
raise ValueError("provider must be a non-empty string")
if not model:
raise ValueError("model must be a non-empty string")
provider_lower = provider.strip().lower()
model_lower = model.strip().lower()
provider_models = self._entries.get(provider_lower, {})
for key, entry in provider_models.items():
if key.lower() == model_lower:
return entry
return self.DEFAULT_COST
def list_providers(self) -> list[str]:
"""Return sorted list of providers with cost data."""
return sorted(self._entries.keys())
def list_models(self, provider: str) -> list[str]:
"""Return sorted list of models for a provider.
Args:
provider: Provider name.
Returns:
List of model names, empty if provider unknown.
Raises:
ValueError: If provider is empty.
"""
if not provider:
raise ValueError("provider must be a non-empty string")
return sorted(self._entries.get(provider.lower(), {}).keys())
+367
View File
@@ -0,0 +1,367 @@
"""Cost tracking and budget enforcement for provider usage.
Tracks token consumption and monetary costs per plan execution,
enforces budget limits, and emits warnings/blocks when thresholds
are approached or exceeded.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import date, datetime
from enum import StrEnum
from typing import ClassVar
from cleveragents.domain.models.core.cost_metadata import (
BudgetExhaustionEvent,
CostMetadata,
)
from cleveragents.providers.cost_table import CostEntry, ProviderCostTable
logger = logging.getLogger(__name__)
class BudgetStatus(StrEnum):
"""Budget enforcement status levels."""
UNDER_BUDGET = "under_budget"
WARNING = "warning"
EXCEEDED = "exceeded"
# Severity ordering for BudgetStatus (higher = worse)
_BUDGET_SEVERITY: dict[BudgetStatus, int] = {
BudgetStatus.UNDER_BUDGET: 0,
BudgetStatus.WARNING: 1,
BudgetStatus.EXCEEDED: 2,
}
@dataclass(frozen=True, slots=True)
class BudgetCheckResult:
"""Result of a budget check operation.
Attributes:
status: Current budget status.
used: Amount used so far (USD).
limit: Budget limit (USD). None if unlimited.
remaining: Amount remaining (USD). None if unlimited.
budget_type: Which budget was checked ('plan' or 'daily').
"""
status: BudgetStatus
used: float
limit: float | None
remaining: float | None
budget_type: str
class CostTracker:
"""Track costs and enforce budget limits for provider usage.
Manages per-plan and per-day budget tracking with configurable
warning and blocking thresholds. Uses dependency injection for
the cost table.
Attributes:
WARN_THRESHOLD: Fraction of budget at which warnings are emitted.
BLOCK_THRESHOLD: Fraction of budget at which requests are blocked.
"""
WARN_THRESHOLD: ClassVar[float] = 0.9
BLOCK_THRESHOLD: ClassVar[float] = 1.0
def __init__(
self,
*,
budget_per_plan: float | None = None,
budget_per_day: float | None = None,
cost_table: ProviderCostTable | None = None,
) -> None:
"""Initialize the cost tracker.
Args:
budget_per_plan: Maximum USD spend per plan (None = unlimited).
budget_per_day: Maximum USD spend per day (None = unlimited).
cost_table: Provider cost lookup table (uses defaults if None).
Raises:
ValueError: If budget values are negative.
"""
if budget_per_plan is not None and budget_per_plan < 0.0:
raise ValueError(f"budget_per_plan must be >= 0, got {budget_per_plan}")
if budget_per_day is not None and budget_per_day < 0.0:
raise ValueError(f"budget_per_day must be >= 0, got {budget_per_day}")
self._budget_per_plan = budget_per_plan
self._budget_per_day = budget_per_day
self._cost_table = cost_table or ProviderCostTable()
self._daily_costs: dict[str, float] = {}
@property
def budget_per_plan(self) -> float | None:
"""Return the per-plan budget limit."""
return self._budget_per_plan
@property
def budget_per_day(self) -> float | None:
"""Return the per-day budget limit."""
return self._budget_per_day
def get_cost_entry(self, provider: str, model: str) -> CostEntry:
"""Look up the cost entry for a provider/model.
Args:
provider: Provider name.
model: Model identifier.
Returns:
The cost entry for the provider/model pair.
Raises:
ValueError: If provider or model is empty.
"""
if not provider:
raise ValueError("provider must be a non-empty string")
if not model:
raise ValueError("model must be a non-empty string")
return self._cost_table.get_cost_entry(provider, model)
def estimate_cost(
self,
provider: str,
model: str,
input_tokens: int,
output_tokens: int,
) -> float:
"""Estimate cost for a token usage.
Args:
provider: Provider name.
model: Model identifier.
input_tokens: Number of input tokens.
output_tokens: Number of output tokens.
Returns:
Estimated cost in USD.
Raises:
ValueError: If arguments are invalid.
"""
if not provider:
raise ValueError("provider must be a non-empty string")
if not model:
raise ValueError("model must be a non-empty string")
if input_tokens < 0:
raise ValueError(f"input_tokens must be >= 0, got {input_tokens}")
if output_tokens < 0:
raise ValueError(f"output_tokens must be >= 0, got {output_tokens}")
entry = self._cost_table.get_cost_entry(provider, model)
return entry.estimate_cost(input_tokens, output_tokens)
def record_usage(
self,
cost_metadata: CostMetadata,
*,
provider: str,
model: str,
input_tokens: int,
output_tokens: int,
) -> BudgetCheckResult:
"""Record usage and check budget status.
Updates cost_metadata in-place with token/cost data.
Also updates internal daily cost tracking.
Records budget exhaustion events when thresholds are crossed.
Args:
cost_metadata: The plan's cost metadata to update.
provider: Provider name.
model: Model identifier.
input_tokens: Input tokens used.
output_tokens: Output tokens used.
Returns:
Budget check result after recording usage.
Raises:
ValueError: If arguments are invalid.
"""
if not provider:
raise ValueError("provider must be a non-empty string")
if not model:
raise ValueError("model must be a non-empty string")
if input_tokens < 0:
raise ValueError(f"input_tokens must be >= 0, got {input_tokens}")
if output_tokens < 0:
raise ValueError(f"output_tokens must be >= 0, got {output_tokens}")
cost = self.estimate_cost(provider, model, input_tokens, output_tokens)
cost_metadata.record_usage(
input_tokens=input_tokens,
output_tokens=output_tokens,
cost=cost,
provider=provider,
)
today_key = date.today().isoformat()
self._daily_costs[today_key] = self._daily_costs.get(today_key, 0.0) + cost
if self._budget_per_plan is not None:
cost_metadata.budget_remaining = max(
0.0, self._budget_per_plan - cost_metadata.total_cost
)
result = self._check_budgets(cost_metadata, provider, model)
return result
def check_plan_budget(self, cost_metadata: CostMetadata) -> BudgetCheckResult:
"""Check the per-plan budget without recording usage.
Args:
cost_metadata: The plan's cost metadata.
Returns:
Budget check result for the plan budget.
"""
return self._evaluate_budget(
used=cost_metadata.total_cost,
limit=self._budget_per_plan,
budget_type="plan",
)
def check_daily_budget(self) -> BudgetCheckResult:
"""Check the per-day budget without recording usage.
Returns:
Budget check result for the daily budget.
"""
today_key = date.today().isoformat()
used = self._daily_costs.get(today_key, 0.0)
return self._evaluate_budget(
used=used,
limit=self._budget_per_day,
budget_type="daily",
)
def get_daily_spend(self) -> float:
"""Return total spend for the current day."""
today_key = date.today().isoformat()
return self._daily_costs.get(today_key, 0.0)
def _check_budgets(
self,
cost_metadata: CostMetadata,
provider: str,
model: str,
) -> BudgetCheckResult:
"""Check both plan and daily budgets, return worst status."""
plan_result = self.check_plan_budget(cost_metadata)
daily_result = self.check_daily_budget()
worst = plan_result
if _BUDGET_SEVERITY.get(daily_result.status, 0) > _BUDGET_SEVERITY.get(
worst.status, 0
):
worst = daily_result
if plan_result.status == BudgetStatus.WARNING:
logger.warning(
"Plan budget warning: %.2f%% used (%.4f / %.4f USD)",
(plan_result.used / plan_result.limit * 100)
if plan_result.limit
else 0,
plan_result.used,
plan_result.limit or 0,
)
if daily_result.status == BudgetStatus.WARNING:
logger.warning(
"Daily budget warning: %.2f%% used (%.4f / %.4f USD)",
(daily_result.used / daily_result.limit * 100)
if daily_result.limit
else 0,
daily_result.used,
daily_result.limit or 0,
)
if plan_result.status == BudgetStatus.EXCEEDED:
self._record_exhaustion_event(cost_metadata, "plan", provider, model)
if daily_result.status == BudgetStatus.EXCEEDED:
self._record_exhaustion_event(cost_metadata, "daily", provider, model)
return worst
def _evaluate_budget(
self,
*,
used: float,
limit: float | None,
budget_type: str,
) -> BudgetCheckResult:
"""Evaluate budget status for a given usage and limit."""
if limit is None:
return BudgetCheckResult(
status=BudgetStatus.UNDER_BUDGET,
used=used,
limit=None,
remaining=None,
budget_type=budget_type,
)
if limit <= 0.0:
return BudgetCheckResult(
status=BudgetStatus.EXCEEDED,
used=used,
limit=limit,
remaining=0.0,
budget_type=budget_type,
)
ratio = used / limit
remaining = max(0.0, limit - used)
if ratio >= self.BLOCK_THRESHOLD:
status = BudgetStatus.EXCEEDED
elif ratio >= self.WARN_THRESHOLD:
status = BudgetStatus.WARNING
else:
status = BudgetStatus.UNDER_BUDGET
return BudgetCheckResult(
status=status,
used=used,
limit=limit,
remaining=remaining,
budget_type=budget_type,
)
def _record_exhaustion_event(
self,
cost_metadata: CostMetadata,
budget_type: str,
provider: str,
model: str,
) -> None:
"""Persist a budget exhaustion event to cost metadata."""
limit = self._budget_per_plan if budget_type == "plan" else self._budget_per_day
if limit is None:
return
event = BudgetExhaustionEvent(
timestamp=datetime.now(),
budget_type=budget_type,
limit=limit,
used=cost_metadata.total_cost,
provider=provider,
model=model,
)
cost_metadata.budget_exhaustion_events.append(event)
logger.warning(
"Budget exhausted: %s budget %.4f USD exceeded "
"(used: %.4f, provider: %s, model: %s)",
budget_type,
limit,
cost_metadata.total_cost,
provider,
model,
)
@@ -0,0 +1,206 @@
"""Provider fallback selection logic.
Selects providers based on required capabilities and budget availability,
skipping providers that lack tool calling, streaming, or other features.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import ClassVar
from cleveragents.providers.cost_tracker import BudgetStatus, CostTracker
from cleveragents.providers.registry import (
ProviderCapabilities,
ProviderInfo,
ProviderRegistry,
ProviderType,
)
logger = logging.getLogger(__name__)
@dataclass(frozen=True, slots=True)
class FallbackResult:
"""Result of a fallback provider selection.
Attributes:
provider_type: Selected provider type, or None if no suitable
provider found.
provider_info: Provider info for the selected provider.
skipped: List of (provider_type, reason) pairs for skipped providers.
"""
provider_type: ProviderType | None
provider_info: ProviderInfo | None
skipped: list[tuple[str, str]]
class FallbackSelector:
"""Select providers using fallback logic with capability filtering.
Iterates through configured providers (in user-defined or default
priority order) and selects the first that meets all required
capabilities and is within budget.
Example::
selector = FallbackSelector(
registry=registry,
cost_tracker=cost_tracker,
fallback_providers=["openai", "anthropic", "groq"],
)
result = selector.select(
require_tool_calls=True,
require_streaming=True,
)
if result.provider_type:
# Use result.provider_type
...
"""
DEFAULT_FALLBACK_ORDER: ClassVar[list[str]] = [
"openai",
"anthropic",
"google",
"azure",
"openrouter",
"groq",
"together",
"cohere",
]
def __init__(
self,
*,
registry: ProviderRegistry,
cost_tracker: CostTracker | None = None,
fallback_providers: list[str] | None = None,
) -> None:
"""Initialize the fallback selector.
Args:
registry: Provider registry for capability lookups.
cost_tracker: Optional cost tracker for budget checks.
fallback_providers: Ordered list of provider names to try.
Uses DEFAULT_FALLBACK_ORDER if not specified.
Raises:
TypeError: If registry is None.
ValueError: If fallback_providers contains empty strings.
"""
if registry is None:
raise TypeError("registry must not be None")
if fallback_providers is not None:
for idx, name in enumerate(fallback_providers):
if not name or not name.strip():
raise ValueError(
f"fallback_providers[{idx}] must be a non-empty string"
)
self._registry = registry
self._cost_tracker = cost_tracker
self._fallback_order = (
[p.strip().lower() for p in fallback_providers]
if fallback_providers
else list(self.DEFAULT_FALLBACK_ORDER)
)
def select(
self,
*,
require_tool_calls: bool = False,
require_streaming: bool = False,
require_vision: bool = False,
require_json_mode: bool = False,
) -> FallbackResult:
"""Select the best available provider matching requirements.
Iterates through the fallback order, skipping providers that:
- Are not configured (no API key)
- Lack required capabilities
- Would exceed budget limits
Args:
require_tool_calls: Require tool/function calling support.
require_streaming: Require streaming response support.
require_vision: Require image/vision input support.
require_json_mode: Require structured JSON output.
Returns:
FallbackResult with the selected provider or None.
"""
skipped: list[tuple[str, str]] = []
for provider_name in self._fallback_order:
info = self._registry.get_provider_info(provider_name)
if info is None:
skipped.append((provider_name, "unknown provider type"))
continue
if not info.is_configured:
skipped.append((provider_name, "not configured (no API key)"))
continue
caps = info.capabilities
skip_reason = self._check_capabilities(
caps,
require_tool_calls=require_tool_calls,
require_streaming=require_streaming,
require_vision=require_vision,
require_json_mode=require_json_mode,
)
if skip_reason:
skipped.append((provider_name, skip_reason))
continue
if self._cost_tracker is not None:
daily_check = self._cost_tracker.check_daily_budget()
if daily_check.status == BudgetStatus.EXCEEDED:
skipped.append((provider_name, "daily budget exceeded"))
continue
logger.info(
"Fallback selected provider: %s (skipped %d)",
provider_name,
len(skipped),
)
return FallbackResult(
provider_type=info.provider_type,
provider_info=info,
skipped=skipped,
)
logger.warning(
"No suitable fallback provider found (tried %d providers)",
len(self._fallback_order),
)
return FallbackResult(
provider_type=None,
provider_info=None,
skipped=skipped,
)
def _check_capabilities(
self,
caps: ProviderCapabilities,
*,
require_tool_calls: bool,
require_streaming: bool,
require_vision: bool,
require_json_mode: bool,
) -> str | None:
"""Check if capabilities meet requirements.
Returns:
Reason string if capabilities are insufficient, None if OK.
"""
if require_tool_calls and not caps.supports_tool_calls:
return "missing capability: tool_calls"
if require_streaming and not caps.supports_streaming:
return "missing capability: streaming"
if require_vision and not caps.supports_vision:
return "missing capability: vision"
if require_json_mode and not caps.supports_json_mode:
return "missing capability: json_mode"
return None
+29
View File
@@ -294,3 +294,32 @@ LockModel # noqa: B018, F821
LockConflictError # noqa: B018, F821
LockExpiredError # noqa: B018, F821
_check_stale_locks # noqa: B018, F821
# Cost controls (M4) — public API surface used by CLI, tests, and benchmarks
CostEntry # noqa: B018, F821
ProviderCostTable # noqa: B018, F821
CostTracker # noqa: B018, F821
BudgetStatus # noqa: B018, F821
BudgetCheckResult # noqa: B018, F821
FallbackSelector # noqa: B018, F821
FallbackResult # noqa: B018, F821
CostMetadata # noqa: B018, F821
BudgetExhaustionEvent # noqa: B018, F821
budget_per_plan # noqa: B018, F821
budget_per_day # noqa: B018, F821
fallback_providers # noqa: B018, F821
cost_metadata # noqa: B018, F821
budget_remaining # noqa: B018, F821
input_tokens # noqa: B018, F821
output_tokens # noqa: B018, F821
provider_costs # noqa: B018, F821
budget_exhaustion_events # noqa: B018, F821
as_display_dict # noqa: B018, F821
record_usage # noqa: B018, F821
estimate_cost # noqa: B018, F821
get_cost_entry # noqa: B018, F821
list_providers # noqa: B018, F821
list_models # noqa: B018, F821
check_plan_budget # noqa: B018, F821
check_daily_budget # noqa: B018, F821
get_daily_spend # noqa: B018, F821