Files
cleveragents-core/features/steps/coverage_boost_steps.py
T
aditya 50680612d5 feat(estimation): add cost and risk estimation actor
Implemented optional estimation_actor role for cost, risk, and duration
estimation during plan lifecycle. Estimates are persisted to plan metadata
and surfaced in CLI output. Key implementation details:

- EstimationOutput (Pydantic models): CostEstimate with currency, token
  estimates, and confidence ranges; RiskScore with 0-100 scale and factors;
  DurationEstimate with min/expected/max seconds. All include confidence
  levels and validation. EstimationSkipped records when estimation is opted out.

- EstimationService: stateless async service that invokes estimation actor
  (stub implementation for M6). Handles actor output parsing, error recovery,
  and fallback to EstimationSkipped on failure.

- Integration: Plan model gains estimation_output and estimation_skipped fields.
  LifecyclePlanModel adds JSON columns for persistence. PlanLifecycleService
  invokes estimation during use_action unless skip_estimation is true.

- CLI: --no-estimate flag added to 'agents plan use'. plan status displays
  cost (USD with token estimates), risk (score/100 with confidence), and
  duration (seconds with ranges) in rich format.

- Database: Alembic migration m6_003_estimation_metadata adds
  estimation_output_json and estimation_skipped_json columns to v3_plans.

- Tests: 44 BDD scenarios (features/estimation.feature) covering validation,
  lifecycle integration, persistence, CLI display, and edge cases. 18 Robot
  Framework smoke tests. 19 ASV benchmark suites for schema, serialization,
  validation, and plan integration performance.

- Documentation: docs/reference/estimation.md with schema, configuration,
  and examples. plan_cli.md updated with --no-estimate flag usage.

ISSUES CLOSED: #209
2026-03-19 15:07:05 +00:00

147 lines
4.7 KiB
Python

"""Step definitions for coverage boost tests."""
import os
from behave import then, when
from cleveragents.config.settings import Settings
from cleveragents.platform import ensure_cli_importable
@when("I create a Settings instance")
def step_create_settings(context):
"""Create a Settings instance."""
# Clear any existing singleton
if hasattr(Settings, "_instance"):
Settings.reset_instance()
# Remove database URL env vars so we test the actual pydantic defaults,
# not the per-scenario temp paths injected by environment.py.
for key in ("CLEVERAGENTS_DATABASE_URL", "CLEVERAGENTS_TEST_DATABASE_URL"):
os.environ.pop(key, None)
context.settings = Settings()
@then("it should have default values")
def step_check_default_values(context):
"""Check Settings has default values."""
assert context.settings.server_host == "0.0.0.0"
assert context.settings.server_port == 8080
assert context.settings.database_url == "sqlite:///cleveragents.db"
assert context.settings.debug_log_level == "INFO"
assert context.settings.storage_base_path.name == "data"
@when("I check if Settings is in production mode")
def step_check_production_mode(context):
"""Check production mode."""
settings = Settings()
context.is_production = settings.is_production
@then("it should return the correct production status")
def step_verify_production_status(context):
"""Verify production status."""
# Default should be production (debug_enabled=False, server_reload=False)
assert context.is_production
@when("I get the database URL from Settings")
def step_get_database_url(context):
"""Get database URL."""
# Clear singleton and database URL env vars so we test the actual
# pydantic defaults, not per-scenario temp paths from environment.py.
if hasattr(Settings, "_instance"):
Settings.reset_instance()
for key in ("CLEVERAGENTS_DATABASE_URL", "CLEVERAGENTS_TEST_DATABASE_URL"):
os.environ.pop(key, None)
settings = Settings()
context.db_url = settings.get_database_url()
context.test_db_url = settings.get_database_url(test=True)
@then("it should return the configured database URL")
def step_check_database_url(context):
"""Check database URL."""
assert context.db_url == "sqlite:///cleveragents.db"
assert context.test_db_url == "sqlite:///cleveragents_test.db"
@when("I check if any provider is configured in Settings")
def step_check_provider_configured(context):
"""Check if provider is configured."""
import shutil
import tempfile
from pathlib import Path
# Clear any existing settings
from cleveragents.config import settings as settings_module
from cleveragents.config.settings import Settings
settings_module._settings = None
# Store current env vars
stored_vars = {}
for key in [
"OPENAI_API_KEY",
"ANTHROPIC_API_KEY",
"GOOGLE_API_KEY",
"AZURE_API_KEY",
"OPENROUTER_API_KEY",
"GEMINI_API_KEY",
"HF_TOKEN",
]:
stored_vars[key] = os.environ.get(key)
os.environ.pop(key, None)
# Move .env file temporarily if it exists
env_file = Path(".env")
temp_env = None
if env_file.exists():
temp_env = Path(tempfile.mktemp(suffix=".env"))
shutil.move(str(env_file), str(temp_env))
try:
# Test with no providers
settings1 = Settings()
context.no_provider = settings1.has_provider_configured()
# Test with a provider
os.environ["OPENAI_API_KEY"] = "test-key"
# Force new settings instance
settings_module._settings = None
settings2 = Settings()
context.with_provider = settings2.has_provider_configured()
finally:
# Restore .env file
if temp_env and temp_env.exists():
shutil.move(str(temp_env), str(env_file))
# Clean up and restore env vars
os.environ.pop("OPENAI_API_KEY", None)
for key, val in stored_vars.items():
if val:
os.environ[key] = val
settings_module._settings = None
@then("it should return the provider status")
def step_verify_provider_status(context):
"""Verify provider status."""
assert not context.no_provider
assert context.with_provider
@when("I import and use the platform module")
def step_use_platform_module(context):
"""Use platform module."""
context.cli_module = ensure_cli_importable()
@then("ensure_cli_importable should work")
def step_check_cli_importable(context):
"""Check ensure_cli_importable worked."""
assert context.cli_module is not None
assert context.cli_module.__name__ == "cleveragents.cli"
assert hasattr(context.cli_module, "main")
assert hasattr(context.cli_module, "app")