cleveragents-core/benchmarks/validation_pipeline_bench.py

"""ASV benchmarks for the validation pipeline.

Measures the performance of:
- Pipeline creation and command sorting
- Pipeline execution with passing validations
- Summary building and aggregation
"""

from __future__ import annotations

import sys
from pathlib import Path
from typing import Any

try:
    from cleveragents.application.services.validation_pipeline import (
        ValidationCommand,
        ValidationPipeline,
        ValidationResult,
        ValidationSummary,
    )
    from cleveragents.domain.models.core.tool import ValidationMode
except ModuleNotFoundError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
    from cleveragents.application.services.validation_pipeline import (
        ValidationCommand,
        ValidationPipeline,
        ValidationResult,
        ValidationSummary,
    )
    from cleveragents.domain.models.core.tool import ValidationMode

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _mock_executor(validation_name: str, arguments: dict[str, Any]) -> dict[str, Any]:
    """Trivial executor that always passes."""
    return {"passed": True, "message": f"{validation_name} ok"}


def _build_commands(count: int) -> list[ValidationCommand]:
    """Build a list of N validation commands across multiple resources."""
    commands: list[ValidationCommand] = []
    for i in range(count):
        resource_name = f"resource-{i % 5}"
        mode = ValidationMode.REQUIRED if i % 3 else ValidationMode.INFORMATIONAL
        commands.append(
            ValidationCommand(
                validation_name=f"check-{i:04d}",
                resource_id=f"R{i:05d}",
                resource_name=resource_name,
                mode=mode,
                arguments={"index": i},
                timeout_seconds=30.0,
            )
        )
    return commands


# ---------------------------------------------------------------------------
# Benchmark suites
# ---------------------------------------------------------------------------


class ValidationPipelineCreationSuite:
    """Benchmarks for pipeline instantiation and command sorting."""

    timeout = 60

    def setup(self) -> None:
        self.commands_10 = _build_commands(10)
        self.commands_100 = _build_commands(100)

    def time_create_pipeline_10_commands(self) -> None:
        """Benchmark creating a pipeline with 10 commands."""
        ValidationPipeline(
            commands=self.commands_10,
            executor=_mock_executor,
            max_workers=4,
        )

    def time_create_pipeline_100_commands(self) -> None:
        """Benchmark creating a pipeline with 100 commands."""
        ValidationPipeline(
            commands=self.commands_100,
            executor=_mock_executor,
            max_workers=4,
        )

    def time_sort_100_commands(self) -> None:
        """Benchmark sorting 100 commands."""
        ValidationPipeline._sort_commands(self.commands_100)


class ValidationPipelineExecutionSuite:
    """Benchmarks for pipeline execution with passing validations."""

    timeout = 120

    def setup(self) -> None:
        self.commands_10 = _build_commands(10)
        self.commands_50 = _build_commands(50)

    def time_run_10_validations(self) -> None:
        """Benchmark running 10 validations."""
        pipeline = ValidationPipeline(
            commands=self.commands_10,
            executor=_mock_executor,
            max_workers=4,
        )
        pipeline.run()

    def time_run_50_validations_sequential(self) -> None:
        """Benchmark running 50 validations with max_workers=1."""
        pipeline = ValidationPipeline(
            commands=self.commands_50,
            executor=_mock_executor,
            max_workers=1,
        )
        pipeline.run()

    def time_run_10_validations_for_plan(self) -> None:
        """Benchmark run_for_plan with 10 validations."""
        pipeline = ValidationPipeline(
            commands=self.commands_10,
            executor=_mock_executor,
            max_workers=4,
        )
        metadata: dict[str, Any] = {}
        pipeline.run_for_plan(plan_metadata=metadata)


class ValidationPipelineSummarySuite:
    """Benchmarks for summary construction and grouping."""

    timeout = 60

    def setup(self) -> None:
        self.results: list[ValidationResult] = []
        for i in range(50):
            self.results.append(
                ValidationResult(
                    validation_name=f"check-{i:04d}",
                    resource_id=f"R{i:05d}",
                    resource_name=f"resource-{i % 5}",
                    mode=(
                        ValidationMode.REQUIRED
                        if i % 3
                        else ValidationMode.INFORMATIONAL
                    ),
                    passed=i % 2 == 0,
                    message=f"Result for check-{i:04d}",
                    data=None,
                    duration_ms=float(i),
                    error=None,
                    timed_out=False,
                )
            )

    def time_group_by_resource_50_results(self) -> None:
        """Benchmark grouping 50 results by resource."""
        ValidationPipeline.group_by_resource(self.results)

    def time_build_summary_model(self) -> None:
        """Benchmark building a ValidationSummary from 50 results."""
        req_p = sum(
            1 for r in self.results if r.mode == ValidationMode.REQUIRED and r.passed
        )
        req_f = sum(
            1
            for r in self.results
            if r.mode == ValidationMode.REQUIRED and not r.passed
        )
        info_p = sum(
            1
            for r in self.results
            if r.mode == ValidationMode.INFORMATIONAL and r.passed
        )
        info_f = sum(
            1
            for r in self.results
            if r.mode == ValidationMode.INFORMATIONAL and not r.passed
        )
        ValidationSummary(
            total=len(self.results),
            required_passed=req_p,
            required_failed=req_f,
            informational_passed=info_p,
            informational_failed=info_f,
            results=self.results,
        )