feat(estimation): implement EstimationReport domain model and estimation_produced decision type #677
@@ -0,0 +1,70 @@
|
||||
"""Add estimation_report column and estimation_produced decision type.
|
||||
|
||||
Adds the estimation_report JSON column to the v3_plans table for
|
||||
persisting structured estimation actor output. Also updates the
|
||||
decisions table CHECK constraint to include the new estimation_produced
|
||||
decision type.
|
||||
|
||||
Revision ID: m6_005_estimation_report_domain
|
||||
Revises: m6_004_resource_type_inherits
|
||||
Create Date: 2026-03-10
|
||||
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "m6_005_estimation_report_domain"
|
||||
down_revision: str | None = "m6_004_resource_type_inherits"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add estimation_report column and update decision type constraint."""
|
||||
|
||||
# Add estimation_report JSON column to v3_plans
|
||||
op.add_column(
|
||||
"v3_plans",
|
||||
sa.Column("estimation_report", sa.Text, nullable=True),
|
||||
)
|
||||
|
||||
# Update decision_type CHECK constraint to include estimation_produced
|
||||
# Use batch mode for SQLite compatibility
|
||||
with op.batch_alter_table("decisions", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("ck_decisions_type", type_="check")
|
||||
batch_op.create_check_constraint(
|
||||
"ck_decisions_type",
|
||||
"decision_type IN ("
|
||||
"'prompt_definition', 'invariant_enforced', "
|
||||
"'strategy_choice', 'implementation_choice', "
|
||||
"'resource_selection', 'subplan_spawn', "
|
||||
"'subplan_parallel_spawn', 'tool_invocation', "
|
||||
"'error_recovery', 'validation_response', "
|
||||
"'user_intervention', 'estimation_produced')",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove estimation_report column and revert decision type constraint."""
|
||||
|
||||
# Drop estimation_report column from v3_plans
|
||||
op.drop_column("v3_plans", "estimation_report")
|
||||
|
||||
# Revert decision_type CHECK constraint to exclude estimation_produced
|
||||
# Use batch mode for SQLite compatibility
|
||||
with op.batch_alter_table("decisions", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("ck_decisions_type", type_="check")
|
||||
batch_op.create_check_constraint(
|
||||
"ck_decisions_type",
|
||||
"decision_type IN ("
|
||||
"'prompt_definition', 'invariant_enforced', "
|
||||
"'strategy_choice', 'implementation_choice', "
|
||||
"'resource_selection', 'subplan_spawn', "
|
||||
"'subplan_parallel_spawn', 'tool_invocation', "
|
||||
"'error_recovery', 'validation_response', "
|
||||
"'user_intervention')",
|
||||
)
|
||||
@@ -320,7 +320,7 @@ FK to `v3_plans.plan_id` (CASCADE delete). Self-referential FKs for
|
||||
| `plan_id` | String(26) | NOT NULL, FK -> v3_plans CASCADE | Parent plan reference |
|
||||
| `parent_decision_id` | String(26) | FK -> decisions SET NULL | Parent decision in tree |
|
||||
| `sequence_number` | Integer | NOT NULL | Monotonic order within plan |
|
||||
| `decision_type` | String(30) | NOT NULL, CHECK (11 enum values) | Classification of the decision |
|
||||
| `decision_type` | String(30) | NOT NULL, CHECK (12 enum values) | Classification of the decision |
|
||||
| `question` | Text | NOT NULL | What question was being answered |
|
||||
| `chosen_option` | Text | NOT NULL | The option that was chosen |
|
||||
| `alternatives_considered_json` | Text | | JSON array of alternative options |
|
||||
|
||||
@@ -32,7 +32,7 @@ targeted correction and replay.
|
||||
|
||||
### Classification
|
||||
|
||||
- **decision_type** — one of the 11 `DecisionType` enum values
|
||||
- **decision_type** — one of the 12 `DecisionType` enum values
|
||||
|
||||
### Content
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ Feature: Consolidated Decision
|
||||
# Feature: Decision domain model
|
||||
# ============================================================
|
||||
|
||||
Scenario: All 11 decision types are defined
|
||||
Then the DecisionType enum should have exactly 11 members
|
||||
Scenario: All 12 decision types are defined
|
||||
Then the DecisionType enum should have exactly 12 members
|
||||
|
||||
|
||||
Scenario: Strategize-phase types are correctly classified
|
||||
@@ -16,7 +16,8 @@ Feature: Consolidated Decision
|
||||
And STRATEGIZE_TYPES should contain "strategy_choice"
|
||||
And STRATEGIZE_TYPES should contain "subplan_spawn"
|
||||
And STRATEGIZE_TYPES should contain "subplan_parallel_spawn"
|
||||
And STRATEGIZE_TYPES should have exactly 5 members
|
||||
And STRATEGIZE_TYPES should contain "estimation_produced"
|
||||
And STRATEGIZE_TYPES should have exactly 6 members
|
||||
|
||||
|
||||
Scenario: Execute-phase types are correctly classified
|
||||
@@ -252,7 +253,7 @@ Feature: Consolidated Decision
|
||||
And as_cli_dict should contain key "parent"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# All 11 decision types can be instantiated
|
||||
# All 12 decision types can be instantiated
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -274,6 +275,7 @@ Feature: Consolidated Decision
|
||||
| error_recovery |
|
||||
| validation_response |
|
||||
| user_intervention |
|
||||
| estimation_produced |
|
||||
|
||||
|
||||
# ============================================================
|
||||
@@ -443,4 +445,3 @@ Feature: Consolidated Decision
|
||||
When I decision persistence create a decision with a 2000 character rationale
|
||||
And I decision persistence round-trip the decision through model_dump
|
||||
Then the decision persistence restored rationale length should be 2000
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
Feature: EstimationReport domain model
|
||||
Tests for the EstimationReport Pydantic model that captures structured
|
||||
multi-dimensional estimation output from the estimation actor.
|
||||
|
||||
Scenario: Create a valid EstimationReport with all required fields
|
||||
When I create an EstimationReport with valid data
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport cost range min should be 0.50
|
||||
And the EstimationReport cost range max should be 2.00
|
||||
And the EstimationReport expected steps should be 5
|
||||
And the EstimationReport expected child plans should be 2
|
||||
And the EstimationReport rollback risk should be 0.25
|
||||
And the EstimationReport confidence should be 0.85
|
||||
|
||||
Scenario: EstimationReport validates cost_range_usd_max >= cost_range_usd_min
|
||||
When I try to create an EstimationReport with max cost less than min cost
|
||||
Then an EstimationReport validation error should be raised
|
||||
And the error should mention "cost_range_usd_max"
|
||||
|
||||
Scenario: EstimationReport validates rollback_risk in [0.0, 1.0] range
|
||||
When I try to create an EstimationReport with rollback_risk 1.5
|
||||
Then an EstimationReport validation error should be raised
|
||||
And the error should mention "rollback_risk"
|
||||
|
||||
Scenario: EstimationReport validates rollback_risk negative value
|
||||
When I try to create an EstimationReport with rollback_risk -0.1
|
||||
Then an EstimationReport validation error should be raised
|
||||
|
||||
Scenario: EstimationReport validates confidence in [0.0, 1.0] range
|
||||
When I try to create an EstimationReport with confidence 1.2
|
||||
Then an EstimationReport validation error should be raised
|
||||
And the error should mention "confidence"
|
||||
|
||||
Scenario: EstimationReport validates confidence negative value
|
||||
When I try to create an EstimationReport with confidence -0.05
|
||||
Then an EstimationReport validation error should be raised
|
||||
|
||||
Scenario: EstimationReport accepts rollback_risk at boundary 0.0
|
||||
When I create an EstimationReport with rollback_risk 0.0
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport rollback risk should be 0.0
|
||||
|
||||
Scenario: EstimationReport accepts rollback_risk at boundary 1.0
|
||||
When I create an EstimationReport with rollback_risk 1.0
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport rollback risk should be 1.0
|
||||
|
||||
Scenario: EstimationReport accepts confidence at boundary 0.0
|
||||
When I create an EstimationReport with confidence 0.0
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport confidence should be 0.0
|
||||
|
||||
Scenario: EstimationReport accepts confidence at boundary 1.0
|
||||
When I create an EstimationReport with confidence 1.0
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport confidence should be 1.0
|
||||
|
||||
Scenario: EstimationReport round-trips through model_dump and model_validate
|
||||
When I create an EstimationReport with valid data
|
||||
And I serialize the EstimationReport with model_dump
|
||||
And I deserialize the EstimationReport with model_validate
|
||||
Then the deserialized EstimationReport should match the original
|
||||
|
||||
Scenario: EstimationReport round-trips through JSON serialization
|
||||
When I create an EstimationReport with valid data
|
||||
And I serialize the EstimationReport to JSON
|
||||
And I deserialize the EstimationReport from JSON
|
||||
Then the deserialized EstimationReport should match the original
|
||||
|
||||
Scenario: EstimationReport with historical basis persists correctly
|
||||
When I create an EstimationReport with 3 historical plan IDs
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport should have 3 historical basis entries
|
||||
|
||||
Scenario: EstimationReport with empty rationale is valid
|
||||
When I create an EstimationReport with empty rationale
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport rationale should be empty
|
||||
|
||||
Scenario: EstimationReport with rationale persists correctly
|
||||
When I create an EstimationReport with rationale "Based on similar complexity"
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport rationale should be "Based on similar complexity"
|
||||
|
||||
Scenario: EstimationReport is immutable (frozen model)
|
||||
When I create an EstimationReport with valid data
|
||||
Then I should not be able to modify the EstimationReport fields
|
||||
|
||||
Scenario: EstimationReport validates negative cost_range_usd_min
|
||||
When I try to create an EstimationReport with cost_range_usd_min -1.0
|
||||
Then an EstimationReport validation error should be raised
|
||||
|
||||
Scenario: EstimationReport validates negative expected_steps
|
||||
When I try to create an EstimationReport with expected_steps -1
|
||||
Then an EstimationReport validation error should be raised
|
||||
|
||||
Scenario: EstimationReport validates negative estimated_duration_minutes
|
||||
When I try to create an EstimationReport with estimated_duration_minutes -5.0
|
||||
Then an EstimationReport validation error should be raised
|
||||
|
||||
Scenario: EstimationReport with zero values is valid
|
||||
When I create an EstimationReport with zero cost and steps
|
||||
Then the EstimationReport should be created successfully
|
||||
And the EstimationReport cost range min should be 0.0
|
||||
And the EstimationReport expected steps should be 0
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Step definitions for estimation_report.feature.
|
||||
|
||||
Tests the EstimationReport Pydantic domain model including validation,
|
||||
serialization, and immutability.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from behave import then, when # type: ignore[import-untyped]
|
||||
from behave.runner import Context
|
||||
from pydantic import ValidationError
|
||||
|
||||
from cleveragents.domain.models.core.estimation import EstimationReport
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_estimation_report(
|
||||
cost_min: float = 0.50,
|
||||
cost_max: float = 2.00,
|
||||
expected_steps: int = 5,
|
||||
expected_child_plans: int = 2,
|
||||
rollback_risk: float = 0.25,
|
||||
confidence: float = 0.85,
|
||||
estimated_duration_minutes: float = 15.0,
|
||||
rationale: str = "Standard complexity estimate",
|
||||
historical_basis: list[str] | None = None,
|
||||
) -> EstimationReport:
|
||||
"""Create an EstimationReport with optional overrides."""
|
||||
return EstimationReport(
|
||||
cost_range_usd_min=cost_min,
|
||||
cost_range_usd_max=cost_max,
|
||||
expected_steps=expected_steps,
|
||||
expected_child_plans=expected_child_plans,
|
||||
rollback_risk=rollback_risk,
|
||||
confidence=confidence,
|
||||
estimated_duration_minutes=estimated_duration_minutes,
|
||||
rationale=rationale,
|
||||
historical_basis=historical_basis or [],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# When steps - Creating EstimationReport instances
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I create an EstimationReport with valid data")
|
||||
def step_create_valid_estimation_report(context: Context) -> None:
|
||||
"""Create a valid EstimationReport and store in context."""
|
||||
context.estimation_report = _make_estimation_report()
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with max cost less than min cost")
|
||||
def step_create_invalid_cost_range(context: Context) -> None:
|
||||
"""Try to create EstimationReport with max < min."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(cost_min=5.0, cost_max=2.0)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with rollback_risk {value:f}")
|
||||
def step_create_invalid_rollback_risk(context: Context, value: float) -> None:
|
||||
"""Try to create EstimationReport with invalid rollback_risk."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(rollback_risk=value)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with confidence {value:f}")
|
||||
def step_create_invalid_confidence(context: Context, value: float) -> None:
|
||||
"""Try to create EstimationReport with invalid confidence."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(confidence=value)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I create an EstimationReport with rollback_risk {value:f}")
|
||||
def step_create_with_rollback_risk(context: Context, value: float) -> None:
|
||||
"""Create EstimationReport with specific rollback_risk."""
|
||||
context.estimation_report = _make_estimation_report(rollback_risk=value)
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when("I create an EstimationReport with confidence {value:f}")
|
||||
def step_create_with_confidence(context: Context, value: float) -> None:
|
||||
"""Create EstimationReport with specific confidence."""
|
||||
context.estimation_report = _make_estimation_report(confidence=value)
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when("I create an EstimationReport with {count:d} historical plan IDs")
|
||||
def step_create_with_historical_basis(context: Context, count: int) -> None:
|
||||
"""Create EstimationReport with historical plan IDs."""
|
||||
historical_ids = [f"01H{i:024d}" for i in range(count)]
|
||||
context.estimation_report = _make_estimation_report(historical_basis=historical_ids)
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when("I create an EstimationReport with empty rationale")
|
||||
def step_create_with_empty_rationale(context: Context) -> None:
|
||||
"""Create EstimationReport with empty rationale."""
|
||||
context.estimation_report = _make_estimation_report(rationale="")
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when('I create an EstimationReport with rationale "{rationale}"')
|
||||
def step_create_with_rationale(context: Context, rationale: str) -> None:
|
||||
"""Create EstimationReport with specific rationale."""
|
||||
context.estimation_report = _make_estimation_report(rationale=rationale)
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with cost_range_usd_min {value:f}")
|
||||
def step_create_invalid_min_cost(context: Context, value: float) -> None:
|
||||
"""Try to create EstimationReport with invalid min cost."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(
|
||||
cost_min=value, cost_max=5.0
|
||||
)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with expected_steps {value:d}")
|
||||
def step_create_invalid_steps(context: Context, value: int) -> None:
|
||||
"""Try to create EstimationReport with invalid expected_steps."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(expected_steps=value)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I try to create an EstimationReport with estimated_duration_minutes {value:f}")
|
||||
def step_create_invalid_duration(context: Context, value: float) -> None:
|
||||
"""Try to create EstimationReport with invalid duration."""
|
||||
try:
|
||||
context.estimation_report = _make_estimation_report(
|
||||
estimated_duration_minutes=value
|
||||
)
|
||||
context.estimation_error = None
|
||||
except ValidationError as e:
|
||||
context.estimation_error = e
|
||||
context.estimation_report = None
|
||||
|
||||
|
||||
@when("I create an EstimationReport with zero cost and steps")
|
||||
def step_create_with_zeros(context: Context) -> None:
|
||||
"""Create EstimationReport with zero values."""
|
||||
context.estimation_report = _make_estimation_report(
|
||||
cost_min=0.0,
|
||||
cost_max=0.0,
|
||||
expected_steps=0,
|
||||
expected_child_plans=0,
|
||||
estimated_duration_minutes=0.0,
|
||||
)
|
||||
context.estimation_error = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialization steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I serialize the EstimationReport with model_dump")
|
||||
def step_serialize_model_dump(context: Context) -> None:
|
||||
"""Serialize EstimationReport using model_dump."""
|
||||
context.serialized_data = context.estimation_report.model_dump()
|
||||
|
||||
|
||||
@when("I deserialize the EstimationReport with model_validate")
|
||||
def step_deserialize_model_validate(context: Context) -> None:
|
||||
"""Deserialize EstimationReport using model_validate."""
|
||||
context.deserialized_report = EstimationReport.model_validate(
|
||||
context.serialized_data
|
||||
)
|
||||
|
||||
|
||||
@when("I serialize the EstimationReport to JSON")
|
||||
def step_serialize_to_json(context: Context) -> None:
|
||||
"""Serialize EstimationReport to JSON string."""
|
||||
context.json_data = context.estimation_report.model_dump_json()
|
||||
|
||||
|
||||
@when("I deserialize the EstimationReport from JSON")
|
||||
def step_deserialize_from_json(context: Context) -> None:
|
||||
"""Deserialize EstimationReport from JSON string."""
|
||||
context.deserialized_report = EstimationReport.model_validate_json(
|
||||
context.json_data
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Then steps - Assertions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@then("the EstimationReport should be created successfully")
|
||||
def step_report_created_successfully(context: Context) -> None:
|
||||
"""Assert EstimationReport was created without errors."""
|
||||
assert context.estimation_report is not None, "EstimationReport should be created"
|
||||
assert context.estimation_error is None, f"Got error: {context.estimation_error}"
|
||||
|
||||
|
||||
@then("an EstimationReport validation error should be raised")
|
||||
def step_validation_error_raised(context: Context) -> None:
|
||||
"""Assert a ValidationError was raised."""
|
||||
assert context.estimation_error is not None, "Expected a ValidationError"
|
||||
assert isinstance(context.estimation_error, ValidationError), (
|
||||
f"Expected ValidationError, got {type(context.estimation_error)}"
|
||||
)
|
||||
# Store in context.error for compatibility with shared steps
|
||||
context.error = context.estimation_error
|
||||
|
||||
|
||||
@then("the EstimationReport cost range min should be {value:f}")
|
||||
def step_check_cost_min(context: Context, value: float) -> None:
|
||||
"""Assert cost_range_usd_min matches expected value."""
|
||||
assert context.estimation_report.cost_range_usd_min == value, (
|
||||
f"Expected {value}, got {context.estimation_report.cost_range_usd_min}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport cost range max should be {value:f}")
|
||||
def step_check_cost_max(context: Context, value: float) -> None:
|
||||
"""Assert cost_range_usd_max matches expected value."""
|
||||
assert context.estimation_report.cost_range_usd_max == value, (
|
||||
f"Expected {value}, got {context.estimation_report.cost_range_usd_max}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport expected steps should be {value:d}")
|
||||
def step_check_expected_steps(context: Context, value: int) -> None:
|
||||
"""Assert expected_steps matches expected value."""
|
||||
assert context.estimation_report.expected_steps == value, (
|
||||
f"Expected {value}, got {context.estimation_report.expected_steps}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport expected child plans should be {value:d}")
|
||||
def step_check_expected_child_plans(context: Context, value: int) -> None:
|
||||
"""Assert expected_child_plans matches expected value."""
|
||||
assert context.estimation_report.expected_child_plans == value, (
|
||||
f"Expected {value}, got {context.estimation_report.expected_child_plans}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport rollback risk should be {value:f}")
|
||||
def step_check_rollback_risk(context: Context, value: float) -> None:
|
||||
"""Assert rollback_risk matches expected value."""
|
||||
assert context.estimation_report.rollback_risk == value, (
|
||||
f"Expected {value}, got {context.estimation_report.rollback_risk}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport confidence should be {value:f}")
|
||||
def step_check_confidence(context: Context, value: float) -> None:
|
||||
"""Assert confidence matches expected value."""
|
||||
assert context.estimation_report.confidence == value, (
|
||||
f"Expected {value}, got {context.estimation_report.confidence}"
|
||||
)
|
||||
|
||||
|
||||
@then("the deserialized EstimationReport should match the original")
|
||||
def step_check_deserialized_matches(context: Context) -> None:
|
||||
"""Assert deserialized report matches original."""
|
||||
assert context.deserialized_report == context.estimation_report, (
|
||||
"Deserialized report should match original"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport should have {count:d} historical basis entries")
|
||||
def step_check_historical_basis_count(context: Context, count: int) -> None:
|
||||
"""Assert historical_basis has expected number of entries."""
|
||||
actual_count = len(context.estimation_report.historical_basis)
|
||||
assert actual_count == count, (
|
||||
f"Expected {count} historical basis entries, got {actual_count}"
|
||||
)
|
||||
|
||||
|
||||
@then("the EstimationReport rationale should be empty")
|
||||
def step_check_rationale_empty(context: Context) -> None:
|
||||
"""Assert rationale is empty string."""
|
||||
assert context.estimation_report.rationale == "", "Rationale should be empty"
|
||||
|
||||
|
||||
@then('the EstimationReport rationale should be "{rationale}"')
|
||||
def step_check_rationale_value(context: Context, rationale: str) -> None:
|
||||
"""Assert rationale matches expected value."""
|
||||
assert context.estimation_report.rationale == rationale, (
|
||||
f"Expected '{rationale}', got '{context.estimation_report.rationale}'"
|
||||
)
|
||||
|
||||
|
||||
@then("I should not be able to modify the EstimationReport fields")
|
||||
def step_check_immutability(context: Context) -> None:
|
||||
"""Assert that EstimationReport fields cannot be modified."""
|
||||
try:
|
||||
context.estimation_report.confidence = 0.5
|
||||
raise AssertionError("Should not be able to modify frozen model")
|
||||
except ValidationError:
|
||||
pass
|
||||
except AttributeError:
|
||||
pass
|
||||
@@ -18,7 +18,7 @@ from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.orm import Session, scoped_session, sessionmaker
|
||||
|
||||
from cleveragents.core.exceptions import DatabaseError
|
||||
from cleveragents.domain.models.core.checkpoint import Checkpoint, CheckpointMetadata
|
||||
@@ -234,7 +234,7 @@ def step_fresh_db(context: Context) -> None:
|
||||
engine = create_engine("sqlite:///:memory:", echo=False)
|
||||
Base.metadata.create_all(engine)
|
||||
context.r2_engine = engine
|
||||
context.r2_session_factory = sessionmaker(bind=engine)
|
||||
context.r2_session_factory = scoped_session(sessionmaker(bind=engine))
|
||||
|
||||
# Pre-create repos used by multiple scenarios
|
||||
context.r2_skill_repo = SkillRepository(
|
||||
|
||||
@@ -21,7 +21,7 @@ Create Child Decision
|
||||
Should Contain ${result.stdout} decision-create-child-ok
|
||||
|
||||
Decision Type Enum Values
|
||||
[Documentation] Verify all 11 decision type enum values
|
||||
[Documentation] Verify all 12 decision type enum values
|
||||
${result}= Run Process ${PYTHON} ${HELPER_SCRIPT} enum_values cwd=${WORKSPACE}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} decision-enums-ok
|
||||
|
||||
@@ -55,7 +55,7 @@ def _test_create_child() -> None:
|
||||
|
||||
|
||||
def _test_enum_values() -> None:
|
||||
"""Verify all 11 decision type enum values."""
|
||||
"""Verify all 12 decision type enum values."""
|
||||
expected = {
|
||||
"prompt_definition",
|
||||
"invariant_enforced",
|
||||
@@ -68,10 +68,11 @@ def _test_enum_values() -> None:
|
||||
"error_recovery",
|
||||
"validation_response",
|
||||
"user_intervention",
|
||||
"estimation_produced",
|
||||
}
|
||||
actual = {dt.value for dt in DecisionType}
|
||||
assert actual == expected, f"Mismatch: {actual.symmetric_difference(expected)}"
|
||||
assert len(DecisionType) == 11
|
||||
assert len(DecisionType) == 12
|
||||
print("decision-enums-ok")
|
||||
|
||||
|
||||
|
||||
@@ -134,6 +134,9 @@ from cleveragents.domain.models.core.escalation import (
|
||||
OperationContext,
|
||||
)
|
||||
|
||||
# Estimation domain models
|
||||
from cleveragents.domain.models.core.estimation import EstimationReport
|
||||
|
||||
# Invariant domain models
|
||||
from cleveragents.domain.models.core.invariant import (
|
||||
Invariant,
|
||||
@@ -374,6 +377,7 @@ __all__ = [
|
||||
"ErrorRecord",
|
||||
"ErrorRecoveryPolicy",
|
||||
"EscalationDecision",
|
||||
"EstimationReport",
|
||||
"ExecutionEnvironment",
|
||||
"FileRecord",
|
||||
"FragmentProvenance",
|
||||
|
||||
@@ -48,6 +48,9 @@ Decision types
|
||||
* - ``user_intervention``
|
||||
- Any
|
||||
- User-provided guidance / correction
|
||||
* - ``estimation_produced``
|
||||
- Strategize
|
||||
- Cost/risk estimation result recorded
|
||||
|
||||
Context snapshots
|
||||
-----------------
|
||||
@@ -105,6 +108,7 @@ class DecisionType(StrEnum):
|
||||
ERROR_RECOVERY = "error_recovery"
|
||||
VALIDATION_RESPONSE = "validation_response"
|
||||
USER_INTERVENTION = "user_intervention"
|
||||
ESTIMATION_PRODUCED = "estimation_produced"
|
||||
|
||||
|
||||
#: Decision types that may only be created during the Strategize phase.
|
||||
@@ -115,6 +119,7 @@ STRATEGIZE_TYPES: frozenset[DecisionType] = frozenset(
|
||||
DecisionType.STRATEGY_CHOICE,
|
||||
DecisionType.SUBPLAN_SPAWN,
|
||||
DecisionType.SUBPLAN_PARALLEL_SPAWN,
|
||||
DecisionType.ESTIMATION_PRODUCED,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Estimation domain models for cost and risk analysis.
|
||||
|
||||
Models the multi-dimensional output of the estimation actor, including
|
||||
cost ranges, step counts, rollback risk, execution time estimates, and
|
||||
confidence scoring.
|
||||
|
||||
Based on docs/specification.md lines 18996-19013 and issue #649.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class EstimationReport(BaseModel):
|
||||
"""Structured multi-dimensional estimation output from estimation actor.
|
||||
|
||||
Captures the full richness of cost/risk estimation beyond a single
|
||||
scalar value. Persisted alongside plans as JSON for auditability
|
||||
and analysis.
|
||||
|
||||
All range constraints (rollback_risk, confidence) are validated
|
||||
at model construction and assignment time.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True, str_strip_whitespace=True)
|
||||
|
||||
# Cost estimation (min/max range in USD)
|
||||
cost_range_usd_min: float = Field(
|
||||
...,
|
||||
ge=0.0,
|
||||
description="Minimum estimated cost in USD",
|
||||
)
|
||||
cost_range_usd_max: float = Field(
|
||||
...,
|
||||
ge=0.0,
|
||||
description="Maximum estimated cost in USD",
|
||||
)
|
||||
|
||||
# Effort estimation
|
||||
expected_steps: int = Field(
|
||||
...,
|
||||
ge=0,
|
||||
description="Expected number of execution steps",
|
||||
)
|
||||
expected_child_plans: int = Field(
|
||||
...,
|
||||
ge=0,
|
||||
description="Expected number of child plans to be spawned",
|
||||
)
|
||||
|
||||
# Risk and confidence metrics (0.0-1.0 range)
|
||||
rollback_risk: float = Field(
|
||||
...,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Estimated probability of rollback (0.0-1.0)",
|
||||
)
|
||||
confidence: float = Field(
|
||||
...,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Confidence in the estimation (0.0-1.0)",
|
||||
)
|
||||
|
||||
# Time estimation
|
||||
estimated_duration_minutes: float = Field(
|
||||
...,
|
||||
ge=0.0,
|
||||
description="Estimated execution time in minutes",
|
||||
)
|
||||
|
||||
# Rationale and provenance
|
||||
rationale: str = Field(
|
||||
default="",
|
||||
description="Human-readable rationale for the estimates",
|
||||
)
|
||||
historical_basis: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Optional list of historical plan IDs used as basis",
|
||||
)
|
||||
|
||||
@field_validator("cost_range_usd_max")
|
||||
@classmethod
|
||||
def _cost_max_gte_min(cls, v: float, info) -> float:
|
||||
"""Ensure max cost is greater than or equal to min cost."""
|
||||
if "cost_range_usd_min" in info.data:
|
||||
min_cost = info.data["cost_range_usd_min"]
|
||||
if v < min_cost:
|
||||
raise ValueError(
|
||||
f"cost_range_usd_max ({v}) must be >= "
|
||||
f"cost_range_usd_min ({min_cost})"
|
||||
)
|
||||
return v
|
||||
|
||||
@field_validator("rollback_risk", "confidence")
|
||||
@classmethod
|
||||
def _validate_probability_range(cls, v: float, info) -> float:
|
||||
"""Validate that probability fields are in [0.0, 1.0] range."""
|
||||
field_name = info.field_name
|
||||
if not 0.0 <= v <= 1.0:
|
||||
raise ValueError(f"{field_name} must be in range [0.0, 1.0], got {v}")
|
||||
return v
|
||||
|
||||
|
||||
__all__ = ["EstimationReport"]
|
||||
@@ -638,6 +638,7 @@ class LifecyclePlanModel(Base): # type: ignore[misc]
|
||||
cost_actual_usd = Column(Float, nullable=True)
|
||||
token_count_input = Column(Integer, nullable=True, default=0)
|
||||
token_count_output = Column(Integer, nullable=True, default=0)
|
||||
estimation_report = Column(Text, nullable=True)
|
||||
|
||||
# Metadata
|
||||
created_by = Column(String(255), nullable=True)
|
||||
|
||||
@@ -5364,7 +5364,7 @@ class CheckpointRepository:
|
||||
rows = (
|
||||
session.query(CheckpointModel)
|
||||
.filter_by(plan_id=plan_id)
|
||||
.order_by(CheckpointModel.created_at)
|
||||
.order_by(CheckpointModel.created_at, CheckpointModel.checkpoint_id)
|
||||
.all()
|
||||
)
|
||||
return [row.to_domain() for row in rows]
|
||||
@@ -5427,7 +5427,7 @@ class CheckpointRepository:
|
||||
rows = (
|
||||
session.query(CheckpointModel)
|
||||
.filter_by(plan_id=plan_id)
|
||||
.order_by(CheckpointModel.created_at)
|
||||
.order_by(CheckpointModel.created_at, CheckpointModel.checkpoint_id)
|
||||
.all()
|
||||
)
|
||||
if len(rows) <= max_checkpoints or len(rows) < 3:
|
||||
|
||||
Reference in New Issue
Block a user