aditya
8da7bd1e56
CI / lint (pull_request) Successful in 16s
CI / typecheck (pull_request) Successful in 26s
CI / security (pull_request) Successful in 22s
CI / quality (pull_request) Successful in 16s
CI / integration_tests (pull_request) Successful in 4m17s
CI / build (pull_request) Successful in 16s
CI / unit_tests (pull_request) Successful in 7m45s
CI / docker (pull_request) Successful in 44s
CI / coverage (pull_request) Successful in 5m38s
135 lines
3.9 KiB
Python
135 lines
3.9 KiB
Python
"""ASV benchmarks for Action YAML schema validation throughput.
|
|
|
|
Measures the performance of:
|
|
- YAML string parsing + schema validation
|
|
- YAML file loading + schema validation
|
|
- Key normalization overhead
|
|
- Invariant normalization overhead
|
|
- model_dump() serialization
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed that lacks the ``action`` sub-package.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Force-reload the top-level package so Python picks up the source tree
|
|
# version (which contains the ``action`` sub-package) instead of the
|
|
# potentially stale installed copy.
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.action.schema import ActionConfigSchema # noqa: E402
|
|
|
|
_MINIMAL_YAML = """\
|
|
name: local/bench-action
|
|
description: Benchmark action
|
|
strategy_actor: openai/gpt-4
|
|
execution_actor: openai/gpt-4
|
|
definition_of_done: Benchmarks pass
|
|
"""
|
|
|
|
_FULL_YAML = """\
|
|
schema_version: "1"
|
|
name: local/bench-full
|
|
description: Full benchmark action
|
|
long_description: |
|
|
This is a detailed description for benchmarking
|
|
with multiple lines of text.
|
|
strategy_actor: openai/gpt-4
|
|
execution_actor: openai/gpt-4
|
|
estimation_actor: local/estimator
|
|
review_actor: local/reviewer
|
|
apply_actor: local/applier
|
|
invariant_actor: local/invariant-resolver
|
|
definition_of_done: All benchmarks pass with zero errors
|
|
reusable: true
|
|
read_only: false
|
|
state: available
|
|
automation_profile: local/trusted
|
|
invariants:
|
|
- "No secrets in code"
|
|
- "All tests must pass"
|
|
- "Coverage above 97%"
|
|
arguments:
|
|
- name: target_coverage
|
|
type: integer
|
|
required: true
|
|
description: Target coverage percentage
|
|
min_value: 1
|
|
max_value: 100
|
|
- name: test_command
|
|
type: string
|
|
required: false
|
|
description: Test framework command
|
|
default: "pytest --cov"
|
|
"""
|
|
|
|
_CAMEL_CASE_YAML = """\
|
|
name: local/bench-camel
|
|
description: camelCase benchmark
|
|
strategyActor: openai/gpt-4
|
|
executionActor: openai/gpt-4
|
|
definitionOfDone: Benchmarks pass
|
|
"""
|
|
|
|
|
|
class ActionSchemaValidationSuite:
|
|
"""Benchmark ActionConfigSchema.from_yaml() throughput."""
|
|
|
|
def time_validate_minimal(self) -> None:
|
|
"""Benchmark minimal YAML validation."""
|
|
ActionConfigSchema.from_yaml(_MINIMAL_YAML)
|
|
|
|
def time_validate_full(self) -> None:
|
|
"""Benchmark fully-populated YAML validation."""
|
|
ActionConfigSchema.from_yaml(_FULL_YAML)
|
|
|
|
def time_validate_camel_case_normalization(self) -> None:
|
|
"""Benchmark YAML with camelCase key normalization."""
|
|
ActionConfigSchema.from_yaml(_CAMEL_CASE_YAML)
|
|
|
|
|
|
class ActionSchemaFileLoadSuite:
|
|
"""Benchmark ActionConfigSchema.from_yaml_file() throughput."""
|
|
|
|
def setup(self) -> None:
|
|
"""Write a temporary YAML file for file-load benchmarks."""
|
|
fd, self._path = tempfile.mkstemp(suffix=".yaml")
|
|
with os.fdopen(fd, "w") as fh:
|
|
fh.write(_FULL_YAML)
|
|
|
|
def teardown(self) -> None:
|
|
"""Remove the temporary file."""
|
|
Path(self._path).unlink(missing_ok=True)
|
|
|
|
def time_load_from_file(self) -> None:
|
|
"""Benchmark loading and validating a YAML file."""
|
|
ActionConfigSchema.from_yaml_file(self._path)
|
|
|
|
|
|
class ActionSchemaSerializationSuite:
|
|
"""Benchmark serialization of a validated ActionConfigSchema."""
|
|
|
|
def setup(self) -> None:
|
|
"""Parse YAML once for serialization benchmarks."""
|
|
self._config = ActionConfigSchema.from_yaml(_FULL_YAML)
|
|
|
|
def time_model_dump(self) -> None:
|
|
"""Benchmark model_dump() serialization."""
|
|
self._config.model_dump()
|
|
|
|
def time_model_dump_json(self) -> None:
|
|
"""Benchmark model_dump_json() JSON serialization."""
|
|
self._config.model_dump_json()
|