test: add TDD bug-capture test for #1025 — plan correct auto-resolve (#1172)
CI / quality (push) Successful in 36s
CI / build (push) Successful in 20s
CI / helm (push) Successful in 22s
CI / lint (push) Successful in 3m21s
CI / typecheck (push) Successful in 3m57s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m7s
CI / unit_tests (push) Successful in 9m18s
CI / docker (push) Successful in 1m18s
CI / coverage (push) Successful in 12m28s
CI / e2e_tests (push) Failing after 19m0s
CI / integration_tests (push) Successful in 24m45s
CI / status-check (push) Failing after 1s
CI / benchmark-publish (push) Successful in 28m39s

## Summary
- add Behave `@tdd_expected_fail` bug-capture scenarios for `plan correct` auto-resolve without `--plan`
- add shared mock fixtures and Robot integration coverage for isolated-container divergence reproducing bug #1025
- add ASV benchmark coverage for active-plan filtering and document the new TDD capture in changelog

## Testing
- nox -s unit_tests -- features/tdd_plan_correct_auto_resolve.feature
- nox -s integration_tests -- --include tdd_bug_1025
- nox -s lint
- nox -s typecheck
- nox -s coverage_report
- nox

Closes #1035

Reviewed-on: #1172
Reviewed-by: Jeffrey Phillips Freeman <jeffrey.freeman@cleverthis.com>
Co-authored-by: Brent Edwards <brent.edwards@cleverthis.com>
Co-committed-by: Brent Edwards <brent.edwards@cleverthis.com>
This commit was merged in pull request #1172.
This commit is contained in:
2026-04-01 00:40:46 +00:00
committed by Forgejo
parent 01b6eb1804
commit 91cc0b1446
7 changed files with 730 additions and 0 deletions
@@ -0,0 +1,100 @@
"""ASV benchmarks for plan correct auto-resolve (issue #1025).
Measures overhead of the ``_resolve_active_plan_id()`` path when the
``plan correct`` command is invoked without a ``--plan`` flag. Benchmarks
the plan listing and filtering that determines the active plan.
"""
from __future__ import annotations
import sys
from datetime import datetime
from pathlib import Path
from typing import ClassVar
from unittest.mock import MagicMock
try:
from cleveragents.domain.models.core.plan import (
NamespacedName,
Plan,
PlanIdentity,
PlanPhase,
PlanTimestamps,
ProcessingState,
ProjectLink,
)
except ModuleNotFoundError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cleveragents.domain.models.core.plan import (
NamespacedName,
Plan,
PlanIdentity,
PlanPhase,
PlanTimestamps,
ProcessingState,
ProjectLink,
)
from ulid import ULID
def _make_plan(phase: PlanPhase, state: ProcessingState) -> Plan:
"""Build a plan with the given phase and state."""
return Plan(
identity=PlanIdentity(plan_id=str(ULID())),
namespaced_name=NamespacedName(
server=None, namespace="local", name="bench-plan"
),
action_name="local/bench-action",
description="Benchmark plan",
definition_of_done=None,
phase=phase,
processing_state=state,
project_links=[ProjectLink(project_name="proj-1")],
strategy_actor="openai/gpt-4",
execution_actor="openai/gpt-4",
created_by=None,
reusable=False,
read_only=False,
timestamps=PlanTimestamps(created_at=datetime.now(), updated_at=datetime.now()),
)
def _build_plan_list(count: int) -> list[Plan]:
"""Build a mixed list of plans with various states.
Approximately half the plans are terminal (APPLIED/CANCELLED),
and half are active (Execute/COMPLETE or Strategize/PROCESSING).
"""
plans: list[Plan] = []
for i in range(count):
if i % 2 == 0:
plans.append(_make_plan(PlanPhase.EXECUTE, ProcessingState.COMPLETE))
else:
plans.append(_make_plan(PlanPhase.APPLY, ProcessingState.APPLIED))
return plans
# ---------------------------------------------------------------------------
# Benchmarks
# ---------------------------------------------------------------------------
class ResolveActivePlanSuite:
"""Benchmark the active plan filtering logic from _resolve_active_plan_id."""
params: ClassVar[list[int]] = [10, 100, 500]
param_names: ClassVar[list[str]] = ["plan_count"]
def setup(self, plan_count: int) -> None:
"""Prepare plan lists of varying size."""
self.plans = _build_plan_list(plan_count)
self.mock_service = MagicMock()
self.mock_service.list_plans.return_value = self.plans
def time_filter_active_plans(self, plan_count: int) -> None:
"""Time the non-terminal filtering done by _resolve_active_plan_id."""
plans = self.mock_service.list_plans()
active = [p for p in plans if not p.is_terminal]
if active:
_ = active[0].identity.plan_id