forked from cleveragents/cleveragents-core
aditya
3f6b255747
Add --no-estimate opt-out flag for plan use command and persist EstimationSkipped reason when estimation is skipped or fails. Add EstimationReport Pydantic domain model (identical to #649) with spec-aligned multi-dimensional output: cost range (min/max USD), expected steps, expected child plans, rollback risk (0.0-1.0), estimated duration in minutes, confidence (0.0-1.0), rationale, and optional historical basis. Add EstimationSkipped domain model with reason, timestamp, and optional actor_name fields for tracking when estimation is skipped via --no-estimate or when the estimation actor fails. Add estimation_produced to DecisionType enum and STRATEGIZE_TYPES. Add Alembic migration m6_006 adding estimation_report_json and estimation_skipped_json columns to v3_plans and updating the ck_decisions_type CHECK constraint. Wire --no-estimate through plan use CLI: when set, creates EstimationSkipped with reason and clears the estimation actor. Update _run_estimation() to persist EstimationSkipped on actor failure instead of only logging. Add docs/reference/estimation.md documenting the estimation feature, EstimationReport schema, EstimationSkipped schema, and CLI examples. Add benchmarks/estimation_actor_bench.py with ASV benchmarks for EstimationReport and EstimationSkipped operations. Add 13 Behave scenarios (estimation_skip.feature) and 6 Robot integration tests (estimation_skip.robot) covering EstimationSkipped model, EstimationReport model, plan field mutual exclusion, and --no-estimate CLI behavior. ISSUES CLOSED: #209
128 lines
4.2 KiB
Python
128 lines
4.2 KiB
Python
"""ASV benchmarks for EstimationReport and EstimationSkipped models.
|
|
|
|
Measures the performance of:
|
|
- EstimationReport model construction (Pydantic validation)
|
|
- EstimationReport.model_dump() serialization
|
|
- EstimationReport.model_dump_json() JSON serialization
|
|
- EstimationReport.model_validate() deserialization
|
|
- EstimationSkipped model construction
|
|
- EstimationSkipped serialization round-trip
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.estimation import (
|
|
EstimationReport,
|
|
EstimationSkipped,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.domain.models.core.estimation import (
|
|
EstimationReport,
|
|
EstimationSkipped,
|
|
)
|
|
|
|
|
|
def _make_full_report() -> EstimationReport:
|
|
"""Create a fully-populated estimation report for benchmarking."""
|
|
return EstimationReport(
|
|
cost_range_usd_min=0.01,
|
|
cost_range_usd_max=0.50,
|
|
expected_steps=10,
|
|
expected_child_plans=3,
|
|
rollback_risk=0.15,
|
|
estimated_duration_minutes=5.5,
|
|
confidence=0.85,
|
|
rationale="Based on similar code review tasks in the last month.",
|
|
historical_basis=("plan-001", "plan-002", "plan-003"),
|
|
)
|
|
|
|
|
|
class EstimationReportConstructionSuite:
|
|
"""Benchmark EstimationReport construction."""
|
|
|
|
def time_minimal_construction(self) -> None:
|
|
"""Benchmark minimal EstimationReport creation."""
|
|
EstimationReport(
|
|
cost_range_usd_min=0.0,
|
|
cost_range_usd_max=0.0,
|
|
expected_steps=0,
|
|
expected_child_plans=0,
|
|
rollback_risk=0.0,
|
|
estimated_duration_minutes=0.0,
|
|
confidence=0.5,
|
|
rationale="Minimal estimation.",
|
|
)
|
|
|
|
def time_full_construction(self) -> None:
|
|
"""Benchmark fully-populated EstimationReport creation."""
|
|
_make_full_report()
|
|
|
|
def time_with_historical_basis(self) -> None:
|
|
"""Benchmark EstimationReport with 50 historical basis entries."""
|
|
EstimationReport(
|
|
cost_range_usd_min=0.10,
|
|
cost_range_usd_max=1.00,
|
|
expected_steps=20,
|
|
expected_child_plans=5,
|
|
rollback_risk=0.3,
|
|
estimated_duration_minutes=15.0,
|
|
confidence=0.7,
|
|
rationale="Large task with extensive history.",
|
|
historical_basis=tuple(f"plan-{i:03d}" for i in range(50)),
|
|
)
|
|
|
|
|
|
class EstimationReportSerializationSuite:
|
|
"""Benchmark EstimationReport serialization."""
|
|
|
|
def setup(self) -> None:
|
|
"""Create objects for serialization benchmarks."""
|
|
self.report = _make_full_report()
|
|
self.report_dict = self.report.model_dump()
|
|
self.report_json = self.report.model_dump_json()
|
|
|
|
def time_model_dump(self) -> None:
|
|
"""Benchmark model_dump() serialization."""
|
|
self.report.model_dump()
|
|
|
|
def time_model_dump_json(self) -> None:
|
|
"""Benchmark model_dump_json() JSON serialization."""
|
|
self.report.model_dump_json()
|
|
|
|
def time_model_validate_dict(self) -> None:
|
|
"""Benchmark model_validate() from dict."""
|
|
EstimationReport.model_validate(self.report_dict)
|
|
|
|
def time_model_validate_json(self) -> None:
|
|
"""Benchmark model_validate_json() from JSON string."""
|
|
EstimationReport.model_validate_json(self.report_json)
|
|
|
|
|
|
class EstimationSkippedSuite:
|
|
"""Benchmark EstimationSkipped construction and serialization."""
|
|
|
|
def time_construction(self) -> None:
|
|
"""Benchmark EstimationSkipped creation."""
|
|
EstimationSkipped(
|
|
reason="User opted out via --no-estimate",
|
|
actor_name="local/estimation-actor",
|
|
)
|
|
|
|
def time_construction_minimal(self) -> None:
|
|
"""Benchmark minimal EstimationSkipped creation."""
|
|
EstimationSkipped(reason="Skipped")
|
|
|
|
def time_round_trip(self) -> None:
|
|
"""Benchmark EstimationSkipped serialization round-trip."""
|
|
skipped = EstimationSkipped(
|
|
reason="Estimation actor failed: timeout",
|
|
actor_name="local/estimation-actor",
|
|
)
|
|
data = skipped.model_dump()
|
|
EstimationSkipped.model_validate(data)
|