80f1664a0d
CI / benchmark-publish (pull_request) Has been skipped
CI / build (pull_request) Successful in 23s
CI / lint (pull_request) Successful in 3m20s
CI / typecheck (pull_request) Successful in 3m54s
CI / security (pull_request) Successful in 4m13s
CI / quality (pull_request) Successful in 4m17s
CI / integration_tests (pull_request) Successful in 9m21s
CI / unit_tests (pull_request) Successful in 9m42s
CI / docker (pull_request) Successful in 14s
CI / e2e_tests (pull_request) Successful in 12m1s
CI / coverage (pull_request) Successful in 11m41s
CI / status-check (pull_request) Successful in 1s
CI / build (push) Successful in 26s
CI / lint (push) Successful in 3m19s
CI / quality (push) Successful in 3m41s
CI / typecheck (push) Successful in 3m56s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 4m2s
CI / integration_tests (push) Successful in 9m3s
CI / unit_tests (push) Successful in 9m24s
CI / docker (push) Successful in 1m9s
CI / e2e_tests (push) Successful in 13m51s
CI / coverage (push) Successful in 11m25s
CI / status-check (push) Successful in 1s
CI / benchmark-publish (push) Failing after 24m52s
CI / benchmark-regression (pull_request) Failing after 37m6s
Implemented Robot Framework integration test suite validating the CI/CD workflow (Specification Example 7). Tests cover ci-profile configuration (automation-profile, format, log level), idempotent resource and project registration with duplicate-detection assertions, three validation tools (ci-lint, ci-typecheck, ci-tests) registration and resource attachment via ToolRegistryService, action creation with typed arguments and invariants per spec Step 2, plan lifecycle with phase-by-phase completion through all phases (strategize, execute, apply) until terminal applied state, and JSON output structure verification including plan_id, phase, state, action, projects, and arguments fields. Post-review fixes applied (round 1): - H1: Fix truncated invariant #2 text to match spec line 39051 - H2: Fix definition_of_done to match spec's 5-bullet-point format - M1: Register all 3 spec validations (ci-lint, ci-typecheck, ci-tests) - M3: Add branch property to resource registration per spec Step 3 - M4: Replace phantom resource_id with real registered resource - M5: Add projects and arguments field assertions to JSON output test - M7: Replace 'polling-based' with 'phase-by-phase' in docs/comments - L1: Use timezone-aware datetime (timezone.utc) - L2: Use tempfile for resource path instead of hardcoded /tmp - L3: Clarify json_output() docstring to reflect as_cli_dict() scope - L4: Tighten attachment count assertion from >= 1 to == 1 (now == 3) Post-review fixes applied (round 2): - M2: Use Setup Test Environment With Database Isolation for pabot safety - L1: Replace remaining hardcoded /tmp paths with tempfile (validation_attach, resource_idempotent duplicate registration) - L2: Fix stale 'Polling-based' comment missed in round 1 - L5: Align action description with spec line 39027 - M1/L3/L6: Document automation_profile propagation gap in use_action() with TODO for when production code wires the field onto Plan - L4: Add comment explaining local actor name deviation from spec Includes CHANGELOG update describing the integration test scope. ISSUES CLOSED: #771
141 lines
4.7 KiB
Python
141 lines
4.7 KiB
Python
"""ASV benchmarks for correction model construction and service operations.
|
|
|
|
Measures the performance of:
|
|
- CorrectionRequest model construction (Pydantic validation)
|
|
- CorrectionImpact / CorrectionResult / CorrectionAttempt construction
|
|
- CorrectionService.request_correction()
|
|
- CorrectionService.analyze_impact()
|
|
- CorrectionService.execute_correction()
|
|
- CorrectionService.list_corrections()
|
|
- CorrectionMode / CorrectionStatus enum access
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.application.services.correction_service import (
|
|
CorrectionService,
|
|
)
|
|
from cleveragents.domain.models.core.correction import (
|
|
CorrectionAttempt,
|
|
CorrectionImpact,
|
|
CorrectionMode,
|
|
CorrectionRequest,
|
|
CorrectionResult,
|
|
CorrectionStatus,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.application.services.correction_service import (
|
|
CorrectionService,
|
|
)
|
|
from cleveragents.domain.models.core.correction import (
|
|
CorrectionAttempt,
|
|
CorrectionImpact,
|
|
CorrectionMode,
|
|
CorrectionRequest,
|
|
CorrectionResult,
|
|
CorrectionStatus,
|
|
)
|
|
|
|
|
|
class CorrectionModelSuite:
|
|
"""Benchmark correction model construction."""
|
|
|
|
def time_correction_request_construction(self) -> None:
|
|
"""Benchmark CorrectionRequest creation with Pydantic validation."""
|
|
CorrectionRequest(
|
|
plan_id="plan-bench-1",
|
|
target_decision_id="DEC-001",
|
|
mode=CorrectionMode.REVERT,
|
|
guidance="Benchmark guidance text",
|
|
)
|
|
|
|
def time_correction_impact_construction(self) -> None:
|
|
"""Benchmark CorrectionImpact creation."""
|
|
CorrectionImpact(
|
|
affected_decisions=["DEC-001", "DEC-002", "DEC-003"],
|
|
affected_files=["src/main.py", "src/utils.py"],
|
|
estimated_cost=1.5,
|
|
risk_level="medium",
|
|
)
|
|
|
|
def time_correction_result_construction(self) -> None:
|
|
"""Benchmark CorrectionResult creation."""
|
|
CorrectionResult(
|
|
correction_id="test-id",
|
|
status=CorrectionStatus.APPLIED,
|
|
new_decisions=["DEC-004"],
|
|
reverted_decisions=["DEC-001"],
|
|
)
|
|
|
|
def time_correction_attempt_construction(self) -> None:
|
|
"""Benchmark CorrectionAttempt creation."""
|
|
CorrectionAttempt(
|
|
correction_id="test-id",
|
|
success=True,
|
|
details={"step": "benchmark"},
|
|
)
|
|
|
|
def time_correction_mode_enum_access(self) -> None:
|
|
"""Benchmark CorrectionMode enum value access."""
|
|
_ = CorrectionMode.REVERT.value
|
|
_ = CorrectionMode.APPEND.value
|
|
|
|
def time_correction_status_enum_access(self) -> None:
|
|
"""Benchmark CorrectionStatus enum value access."""
|
|
_ = CorrectionStatus.PENDING.value
|
|
_ = CorrectionStatus.ANALYZING.value
|
|
_ = CorrectionStatus.EXECUTING.value
|
|
_ = CorrectionStatus.APPLIED.value
|
|
_ = CorrectionStatus.FAILED.value
|
|
_ = CorrectionStatus.CANCELLED.value
|
|
|
|
|
|
class CorrectionServiceSuite:
|
|
"""Benchmark CorrectionService operations."""
|
|
|
|
def setup(self) -> None:
|
|
"""Create a fresh service for each benchmark."""
|
|
self.service = CorrectionService()
|
|
|
|
def time_request_correction(self) -> None:
|
|
"""Benchmark creating a correction request."""
|
|
self.service.request_correction(
|
|
plan_id="plan-bench",
|
|
target_decision_id="DEC-001",
|
|
mode=CorrectionMode.REVERT,
|
|
guidance="Benchmark correction",
|
|
)
|
|
|
|
def time_analyze_impact(self) -> None:
|
|
"""Benchmark impact analysis (stub)."""
|
|
req = self.service.request_correction(
|
|
plan_id="plan-bench",
|
|
target_decision_id="DEC-001",
|
|
mode=CorrectionMode.REVERT,
|
|
guidance="Benchmark analysis",
|
|
)
|
|
self.service.analyze_impact(req.correction_id)
|
|
|
|
def time_execute_correction(self) -> None:
|
|
"""Benchmark correction execution (stub)."""
|
|
req = self.service.request_correction(
|
|
plan_id="plan-bench",
|
|
target_decision_id="DEC-001",
|
|
mode=CorrectionMode.REVERT,
|
|
guidance="Benchmark execution",
|
|
)
|
|
self.service.execute_correction(req.correction_id)
|
|
|
|
def time_list_corrections(self) -> None:
|
|
"""Benchmark listing corrections."""
|
|
self.service.list_corrections()
|
|
|
|
def time_list_corrections_filtered(self) -> None:
|
|
"""Benchmark listing corrections with plan filter."""
|
|
self.service.list_corrections(plan_id="plan-bench")
|