test: add TDD bug-capture test for #1025 — plan correct auto-resolve (#1172)
CI / quality (push) Successful in 36s
CI / build (push) Successful in 20s
CI / helm (push) Successful in 22s
CI / lint (push) Successful in 3m21s
CI / typecheck (push) Successful in 3m57s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m7s
CI / unit_tests (push) Successful in 9m18s
CI / docker (push) Successful in 1m18s
CI / coverage (push) Successful in 12m28s
CI / e2e_tests (push) Failing after 19m0s
CI / integration_tests (push) Successful in 24m45s
CI / status-check (push) Failing after 1s
CI / benchmark-publish (push) Successful in 28m39s
CI / quality (push) Successful in 36s
CI / build (push) Successful in 20s
CI / helm (push) Successful in 22s
CI / lint (push) Successful in 3m21s
CI / typecheck (push) Successful in 3m57s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m7s
CI / unit_tests (push) Successful in 9m18s
CI / docker (push) Successful in 1m18s
CI / coverage (push) Successful in 12m28s
CI / e2e_tests (push) Failing after 19m0s
CI / integration_tests (push) Successful in 24m45s
CI / status-check (push) Failing after 1s
CI / benchmark-publish (push) Successful in 28m39s
## Summary - add Behave `@tdd_expected_fail` bug-capture scenarios for `plan correct` auto-resolve without `--plan` - add shared mock fixtures and Robot integration coverage for isolated-container divergence reproducing bug #1025 - add ASV benchmark coverage for active-plan filtering and document the new TDD capture in changelog ## Testing - nox -s unit_tests -- features/tdd_plan_correct_auto_resolve.feature - nox -s integration_tests -- --include tdd_bug_1025 - nox -s lint - nox -s typecheck - nox -s coverage_report - nox Closes #1035 Reviewed-on: #1172 Reviewed-by: Jeffrey Phillips Freeman <jeffrey.freeman@cleverthis.com> Co-authored-by: Brent Edwards <brent.edwards@cleverthis.com> Co-committed-by: Brent Edwards <brent.edwards@cleverthis.com>
This commit was merged in pull request #1172.
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
"""ASV benchmarks for plan correct auto-resolve (issue #1025).
|
||||
|
||||
Measures overhead of the ``_resolve_active_plan_id()`` path when the
|
||||
``plan correct`` command is invoked without a ``--plan`` flag. Benchmarks
|
||||
the plan listing and filtering that determines the active plan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
try:
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
except ModuleNotFoundError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
from cleveragents.domain.models.core.plan import (
|
||||
NamespacedName,
|
||||
Plan,
|
||||
PlanIdentity,
|
||||
PlanPhase,
|
||||
PlanTimestamps,
|
||||
ProcessingState,
|
||||
ProjectLink,
|
||||
)
|
||||
|
||||
from ulid import ULID
|
||||
|
||||
|
||||
def _make_plan(phase: PlanPhase, state: ProcessingState) -> Plan:
|
||||
"""Build a plan with the given phase and state."""
|
||||
return Plan(
|
||||
identity=PlanIdentity(plan_id=str(ULID())),
|
||||
namespaced_name=NamespacedName(
|
||||
server=None, namespace="local", name="bench-plan"
|
||||
),
|
||||
action_name="local/bench-action",
|
||||
description="Benchmark plan",
|
||||
definition_of_done=None,
|
||||
phase=phase,
|
||||
processing_state=state,
|
||||
project_links=[ProjectLink(project_name="proj-1")],
|
||||
strategy_actor="openai/gpt-4",
|
||||
execution_actor="openai/gpt-4",
|
||||
created_by=None,
|
||||
reusable=False,
|
||||
read_only=False,
|
||||
timestamps=PlanTimestamps(created_at=datetime.now(), updated_at=datetime.now()),
|
||||
)
|
||||
|
||||
|
||||
def _build_plan_list(count: int) -> list[Plan]:
|
||||
"""Build a mixed list of plans with various states.
|
||||
|
||||
Approximately half the plans are terminal (APPLIED/CANCELLED),
|
||||
and half are active (Execute/COMPLETE or Strategize/PROCESSING).
|
||||
"""
|
||||
plans: list[Plan] = []
|
||||
for i in range(count):
|
||||
if i % 2 == 0:
|
||||
plans.append(_make_plan(PlanPhase.EXECUTE, ProcessingState.COMPLETE))
|
||||
else:
|
||||
plans.append(_make_plan(PlanPhase.APPLY, ProcessingState.APPLIED))
|
||||
return plans
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Benchmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ResolveActivePlanSuite:
|
||||
"""Benchmark the active plan filtering logic from _resolve_active_plan_id."""
|
||||
|
||||
params: ClassVar[list[int]] = [10, 100, 500]
|
||||
param_names: ClassVar[list[str]] = ["plan_count"]
|
||||
|
||||
def setup(self, plan_count: int) -> None:
|
||||
"""Prepare plan lists of varying size."""
|
||||
self.plans = _build_plan_list(plan_count)
|
||||
self.mock_service = MagicMock()
|
||||
self.mock_service.list_plans.return_value = self.plans
|
||||
|
||||
def time_filter_active_plans(self, plan_count: int) -> None:
|
||||
"""Time the non-terminal filtering done by _resolve_active_plan_id."""
|
||||
plans = self.mock_service.list_plans()
|
||||
active = [p for p in plans if not p.is_terminal]
|
||||
if active:
|
||||
_ = active[0].identity.plan_id
|
||||
Reference in New Issue
Block a user