Files
cleveragents-core/benchmarks/decision_correction_model_bench.py
CoreRasurae 80f1664a0d
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 23s
CI / lint (pull_request) Successful in 3m20s
CI / typecheck (pull_request) Successful in 3m54s
CI / security (pull_request) Successful in 4m13s
CI / quality (pull_request) Successful in 4m17s
CI / integration_tests (pull_request) Successful in 9m21s
CI / unit_tests (pull_request) Successful in 9m42s
CI / docker (pull_request) Successful in 14s
CI / e2e_tests (pull_request) Successful in 12m1s
CI / coverage (pull_request) Successful in 11m41s
CI / status-check (pull_request) Successful in 1s
CI / build (push) Successful in 26s
CI / lint (push) Successful in 3m19s
CI / quality (push) Successful in 3m41s
CI / typecheck (push) Successful in 3m56s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m2s
CI / integration_tests (push) Successful in 9m3s
CI / unit_tests (push) Successful in 9m24s
CI / docker (push) Successful in 1m9s
CI / e2e_tests (push) Successful in 13m51s
CI / coverage (push) Successful in 11m25s
CI / status-check (push) Successful in 1s
CI / benchmark-publish (push) Failing after 24m52s
CI / benchmark-regression (pull_request) Failing after 37m6s
test(integration): workflow example 7 — CI/CD integration, automated PR review and fix (ci profile)
Implemented Robot Framework integration test suite validating the CI/CD
workflow (Specification Example 7). Tests cover ci-profile configuration
(automation-profile, format, log level), idempotent resource and project
registration with duplicate-detection assertions, three validation tools
(ci-lint, ci-typecheck, ci-tests) registration and resource attachment
via ToolRegistryService, action creation with typed arguments and
invariants per spec Step 2, plan lifecycle with phase-by-phase completion
through all phases (strategize, execute, apply) until terminal applied
state, and JSON output structure verification including plan_id, phase,
state, action, projects, and arguments fields.

Post-review fixes applied (round 1):
- H1: Fix truncated invariant #2 text to match spec line 39051
- H2: Fix definition_of_done to match spec's 5-bullet-point format
- M1: Register all 3 spec validations (ci-lint, ci-typecheck, ci-tests)
- M3: Add branch property to resource registration per spec Step 3
- M4: Replace phantom resource_id with real registered resource
- M5: Add projects and arguments field assertions to JSON output test
- M7: Replace 'polling-based' with 'phase-by-phase' in docs/comments
- L1: Use timezone-aware datetime (timezone.utc)
- L2: Use tempfile for resource path instead of hardcoded /tmp
- L3: Clarify json_output() docstring to reflect as_cli_dict() scope
- L4: Tighten attachment count assertion from >= 1 to == 1 (now == 3)

Post-review fixes applied (round 2):
- M2: Use Setup Test Environment With Database Isolation for pabot safety
- L1: Replace remaining hardcoded /tmp paths with tempfile (validation_attach,
  resource_idempotent duplicate registration)
- L2: Fix stale 'Polling-based' comment missed in round 1
- L5: Align action description with spec line 39027
- M1/L3/L6: Document automation_profile propagation gap in use_action()
  with TODO for when production code wires the field onto Plan
- L4: Add comment explaining local actor name deviation from spec

Includes CHANGELOG update describing the integration test scope.

ISSUES CLOSED: #771
2026-03-26 21:42:37 +00:00

141 lines
4.7 KiB
Python

"""ASV benchmarks for correction model construction and service operations.
Measures the performance of:
- CorrectionRequest model construction (Pydantic validation)
- CorrectionImpact / CorrectionResult / CorrectionAttempt construction
- CorrectionService.request_correction()
- CorrectionService.analyze_impact()
- CorrectionService.execute_correction()
- CorrectionService.list_corrections()
- CorrectionMode / CorrectionStatus enum access
"""
from __future__ import annotations
import sys
from pathlib import Path
try:
from cleveragents.application.services.correction_service import (
CorrectionService,
)
from cleveragents.domain.models.core.correction import (
CorrectionAttempt,
CorrectionImpact,
CorrectionMode,
CorrectionRequest,
CorrectionResult,
CorrectionStatus,
)
except ModuleNotFoundError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cleveragents.application.services.correction_service import (
CorrectionService,
)
from cleveragents.domain.models.core.correction import (
CorrectionAttempt,
CorrectionImpact,
CorrectionMode,
CorrectionRequest,
CorrectionResult,
CorrectionStatus,
)
class CorrectionModelSuite:
"""Benchmark correction model construction."""
def time_correction_request_construction(self) -> None:
"""Benchmark CorrectionRequest creation with Pydantic validation."""
CorrectionRequest(
plan_id="plan-bench-1",
target_decision_id="DEC-001",
mode=CorrectionMode.REVERT,
guidance="Benchmark guidance text",
)
def time_correction_impact_construction(self) -> None:
"""Benchmark CorrectionImpact creation."""
CorrectionImpact(
affected_decisions=["DEC-001", "DEC-002", "DEC-003"],
affected_files=["src/main.py", "src/utils.py"],
estimated_cost=1.5,
risk_level="medium",
)
def time_correction_result_construction(self) -> None:
"""Benchmark CorrectionResult creation."""
CorrectionResult(
correction_id="test-id",
status=CorrectionStatus.APPLIED,
new_decisions=["DEC-004"],
reverted_decisions=["DEC-001"],
)
def time_correction_attempt_construction(self) -> None:
"""Benchmark CorrectionAttempt creation."""
CorrectionAttempt(
correction_id="test-id",
success=True,
details={"step": "benchmark"},
)
def time_correction_mode_enum_access(self) -> None:
"""Benchmark CorrectionMode enum value access."""
_ = CorrectionMode.REVERT.value
_ = CorrectionMode.APPEND.value
def time_correction_status_enum_access(self) -> None:
"""Benchmark CorrectionStatus enum value access."""
_ = CorrectionStatus.PENDING.value
_ = CorrectionStatus.ANALYZING.value
_ = CorrectionStatus.EXECUTING.value
_ = CorrectionStatus.APPLIED.value
_ = CorrectionStatus.FAILED.value
_ = CorrectionStatus.CANCELLED.value
class CorrectionServiceSuite:
"""Benchmark CorrectionService operations."""
def setup(self) -> None:
"""Create a fresh service for each benchmark."""
self.service = CorrectionService()
def time_request_correction(self) -> None:
"""Benchmark creating a correction request."""
self.service.request_correction(
plan_id="plan-bench",
target_decision_id="DEC-001",
mode=CorrectionMode.REVERT,
guidance="Benchmark correction",
)
def time_analyze_impact(self) -> None:
"""Benchmark impact analysis (stub)."""
req = self.service.request_correction(
plan_id="plan-bench",
target_decision_id="DEC-001",
mode=CorrectionMode.REVERT,
guidance="Benchmark analysis",
)
self.service.analyze_impact(req.correction_id)
def time_execute_correction(self) -> None:
"""Benchmark correction execution (stub)."""
req = self.service.request_correction(
plan_id="plan-bench",
target_decision_id="DEC-001",
mode=CorrectionMode.REVERT,
guidance="Benchmark execution",
)
self.service.execute_correction(req.correction_id)
def time_list_corrections(self) -> None:
"""Benchmark listing corrections."""
self.service.list_corrections()
def time_list_corrections_filtered(self) -> None:
"""Benchmark listing corrections with plan filter."""
self.service.list_corrections(plan_id="plan-bench")