"""ASV benchmarks for the validation pipeline. Measures the performance of: - Pipeline creation and command sorting - Pipeline execution with passing validations - Summary building and aggregation """ from __future__ import annotations import sys from pathlib import Path from typing import Any try: from cleveragents.application.services.validation_pipeline import ( ValidationCommand, ValidationPipeline, ValidationResult, ValidationSummary, ) from cleveragents.domain.models.core.tool import ValidationMode except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.application.services.validation_pipeline import ( ValidationCommand, ValidationPipeline, ValidationResult, ValidationSummary, ) from cleveragents.domain.models.core.tool import ValidationMode # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _mock_executor(validation_name: str, arguments: dict[str, Any]) -> dict[str, Any]: """Trivial executor that always passes.""" return {"passed": True, "message": f"{validation_name} ok"} def _build_commands(count: int) -> list[ValidationCommand]: """Build a list of N validation commands across multiple resources.""" commands: list[ValidationCommand] = [] for i in range(count): resource_name = f"resource-{i % 5}" mode = ValidationMode.REQUIRED if i % 3 else ValidationMode.INFORMATIONAL commands.append( ValidationCommand( validation_name=f"check-{i:04d}", resource_id=f"R{i:05d}", resource_name=resource_name, mode=mode, arguments={"index": i}, timeout_seconds=30.0, ) ) return commands # --------------------------------------------------------------------------- # Benchmark suites # --------------------------------------------------------------------------- class ValidationPipelineCreationSuite: """Benchmarks for pipeline instantiation and command sorting.""" timeout = 60 def setup(self) -> None: self.commands_10 = _build_commands(10) self.commands_100 = _build_commands(100) def time_create_pipeline_10_commands(self) -> None: """Benchmark creating a pipeline with 10 commands.""" ValidationPipeline( commands=self.commands_10, executor=_mock_executor, max_workers=4, ) def time_create_pipeline_100_commands(self) -> None: """Benchmark creating a pipeline with 100 commands.""" ValidationPipeline( commands=self.commands_100, executor=_mock_executor, max_workers=4, ) def time_sort_100_commands(self) -> None: """Benchmark sorting 100 commands.""" ValidationPipeline._sort_commands(self.commands_100) class ValidationPipelineExecutionSuite: """Benchmarks for pipeline execution with passing validations.""" timeout = 120 def setup(self) -> None: self.commands_10 = _build_commands(10) self.commands_50 = _build_commands(50) def time_run_10_validations(self) -> None: """Benchmark running 10 validations.""" pipeline = ValidationPipeline( commands=self.commands_10, executor=_mock_executor, max_workers=4, ) pipeline.run() def time_run_50_validations_sequential(self) -> None: """Benchmark running 50 validations with max_workers=1.""" pipeline = ValidationPipeline( commands=self.commands_50, executor=_mock_executor, max_workers=1, ) pipeline.run() def time_run_10_validations_for_plan(self) -> None: """Benchmark run_for_plan with 10 validations.""" pipeline = ValidationPipeline( commands=self.commands_10, executor=_mock_executor, max_workers=4, ) metadata: dict[str, Any] = {} pipeline.run_for_plan(plan_metadata=metadata) class ValidationPipelineSummarySuite: """Benchmarks for summary construction and grouping.""" timeout = 60 def setup(self) -> None: self.results: list[ValidationResult] = [] for i in range(50): self.results.append( ValidationResult( validation_name=f"check-{i:04d}", resource_id=f"R{i:05d}", resource_name=f"resource-{i % 5}", mode=( ValidationMode.REQUIRED if i % 3 else ValidationMode.INFORMATIONAL ), passed=i % 2 == 0, message=f"Result for check-{i:04d}", data=None, duration_ms=float(i), error=None, timed_out=False, ) ) def time_group_by_resource_50_results(self) -> None: """Benchmark grouping 50 results by resource.""" ValidationPipeline.group_by_resource(self.results) def time_build_summary_model(self) -> None: """Benchmark building a ValidationSummary from 50 results.""" req_p = sum( 1 for r in self.results if r.mode == ValidationMode.REQUIRED and r.passed ) req_f = sum( 1 for r in self.results if r.mode == ValidationMode.REQUIRED and not r.passed ) info_p = sum( 1 for r in self.results if r.mode == ValidationMode.INFORMATIONAL and r.passed ) info_f = sum( 1 for r in self.results if r.mode == ValidationMode.INFORMATIONAL and not r.passed ) ValidationSummary( total=len(self.results), required_passed=req_p, required_failed=req_f, informational_passed=info_p, informational_failed=info_f, results=self.results, )