brent.edwards
eb46f0ff54
CI / push-validation (push) Successful in 40s
CI / helm (push) Successful in 44s
CI / build (push) Successful in 1m14s
CI / lint (push) Successful in 1m18s
CI / quality (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 1m19s
CI / typecheck (push) Successful in 2m22s
CI / security (push) Successful in 2m23s
CI / benchmark-regression (push) Failing after 41s
CI / integration_tests (push) Successful in 4m46s
CI / unit_tests (push) Failing after 20m13s
CI / benchmark-publish (push) Failing after 28m28s
CI / coverage (push) Has been cancelled
CI / docker (push) Has been cancelled
CI / status-check (push) Has been cancelled
## Summary This PR fixes issue #10878 where architecture reviews were truncated because the regex pattern for parsing file output would stop at the first ``` encountered in the Markdown report. ## Changes - Change file delimiters from ``` to >>>>>>>/<<<<<<< to avoid Markdown conflicts - Add tier hydration before strategize phase in plan_executor.py - Increase max_tokens to 16384 in llm_actors.py for longer outputs - Increase context_max_tokens_hot from 16000 to 32000 in settings.py - Fix get_hot_view → get_hot_fragments in strategy_actor.py and plan_executor.py - Add opencode to skip directories in context_tier_hydrator.py - Change sandbox output location to plan-output/ directory in plan.py - Add get_context_summary stub method to acms_service.py ## Testing Run architecture review action and verify the output report is complete with all sections. Reviewed-on: #10938
123 lines
3.4 KiB
Python
123 lines
3.4 KiB
Python
"""ASV benchmarks for error recovery patterns.
|
|
|
|
Measures the performance of:
|
|
- Error classification
|
|
- Recovery hint generation
|
|
- ErrorRecord/ErrorHistory model operations
|
|
- ErrorRecoveryService.record_error() with mock lifecycle
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
try:
|
|
from cleveragents.application.services.error_recovery_service import (
|
|
ErrorRecoveryService,
|
|
)
|
|
from cleveragents.domain.models.core.error_recovery import (
|
|
ErrorCategory,
|
|
classify_error,
|
|
get_recovery_hints,
|
|
)
|
|
from cleveragents.domain.models.core.plan import (
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
)
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
from cleveragents.application.services.error_recovery_service import (
|
|
ErrorRecoveryService,
|
|
)
|
|
from cleveragents.domain.models.core.error_recovery import (
|
|
ErrorCategory,
|
|
classify_error,
|
|
get_recovery_hints,
|
|
)
|
|
from cleveragents.domain.models.core.plan import (
|
|
PlanPhase,
|
|
PlanTimestamps,
|
|
ProcessingState,
|
|
)
|
|
|
|
|
|
_PLAN_ID = "01BENCHERR0000000000001"
|
|
|
|
|
|
def _make_plan() -> MagicMock:
|
|
plan = MagicMock()
|
|
plan.identity.plan_id = _PLAN_ID
|
|
plan.phase = PlanPhase.EXECUTE
|
|
plan.processing_state = ProcessingState.ERRORED
|
|
plan.is_terminal = False
|
|
plan.error_message = None
|
|
plan.error_details = None
|
|
plan.timestamps = PlanTimestamps()
|
|
return plan
|
|
|
|
|
|
def _make_service(plan: MagicMock) -> ErrorRecoveryService:
|
|
lifecycle = MagicMock()
|
|
lifecycle.get_plan.return_value = plan
|
|
lifecycle.commit_plan = MagicMock()
|
|
return ErrorRecoveryService(
|
|
lifecycle_service=lifecycle,
|
|
auto_retry_threshold=0.0,
|
|
max_retries=3,
|
|
)
|
|
|
|
|
|
class ClassifySuite:
|
|
"""Benchmark error classification."""
|
|
|
|
def time_classify_transient(self) -> None:
|
|
classify_error("Connection timed out after 30s")
|
|
|
|
def time_classify_validation(self) -> None:
|
|
classify_error("Schema validation failed for field X")
|
|
|
|
def time_classify_unknown(self) -> None:
|
|
classify_error("Something completely novel happened")
|
|
|
|
def time_classify_with_exception_type(self) -> None:
|
|
classify_error("failed", "RateLimitError")
|
|
|
|
|
|
class HintSuite:
|
|
"""Benchmark recovery hint generation."""
|
|
|
|
def time_hints_transient(self) -> None:
|
|
get_recovery_hints(ErrorCategory.TRANSIENT, _PLAN_ID)
|
|
|
|
def time_hints_validation(self) -> None:
|
|
get_recovery_hints(ErrorCategory.VALIDATION, _PLAN_ID)
|
|
|
|
def time_hints_merge_conflict(self) -> None:
|
|
get_recovery_hints(ErrorCategory.MERGE_CONFLICT, _PLAN_ID)
|
|
|
|
|
|
class RecordSuite:
|
|
"""Benchmark error recording with mock service."""
|
|
|
|
def setup(self) -> None:
|
|
self.plan = _make_plan()
|
|
self.service = _make_service(self.plan)
|
|
self._counter = 0
|
|
|
|
def time_record_error(self) -> None:
|
|
self._counter += 1
|
|
self.service.record_error(
|
|
plan_id=_PLAN_ID,
|
|
phase="execute",
|
|
message=f"Timeout #{self._counter}",
|
|
)
|
|
|
|
def time_format_output_plain(self) -> None:
|
|
self.service.format_error_output(_PLAN_ID, fmt="plain")
|
|
|
|
def time_format_output_json(self) -> None:
|
|
self.service.format_error_output(_PLAN_ID, fmt="json")
|