freemo
92c83ecc7e
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / build (pull_request) Successful in 17s
CI / quality (pull_request) Successful in 19s
CI / typecheck (pull_request) Successful in 33s
CI / security (pull_request) Successful in 35s
CI / integration_tests (pull_request) Successful in 2m41s
CI / unit_tests (pull_request) Successful in 6m18s
CI / docker (pull_request) Successful in 1m2s
CI / benchmark-regression (pull_request) Successful in 15m41s
CI / coverage (pull_request) Failing after 20m52s
294 lines
9.7 KiB
Python
294 lines
9.7 KiB
Python
"""ASV benchmarks for Action domain model operations.
|
|
|
|
Measures the performance of:
|
|
- ActionArgument.parse() from colon-delimited strings
|
|
- ActionArgument.coerce_value() type coercion
|
|
- Action.from_config() construction from YAML config dicts
|
|
- Action.as_cli_dict() stable CLI rendering
|
|
- Action.render_template() placeholder substitution
|
|
- ActionArgument.from_mapping() construction from dicts
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cleveragents.domain.models.core.action import (
|
|
Action,
|
|
ActionArgument,
|
|
ArgumentRequirement,
|
|
ArgumentType,
|
|
)
|
|
from cleveragents.domain.models.core.plan import NamespacedName
|
|
except ModuleNotFoundError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src"))
|
|
from cleveragents.domain.models.core.action import (
|
|
Action,
|
|
ActionArgument,
|
|
ArgumentRequirement,
|
|
ArgumentType,
|
|
)
|
|
from cleveragents.domain.models.core.plan import NamespacedName
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_FULL_CONFIG: dict = {
|
|
"name": "local/code-coverage",
|
|
"description": "Increase test coverage to ${target_coverage}%",
|
|
"long_description": "Systematically improve test coverage across the project.",
|
|
"definition_of_done": "Coverage reaches ${target_coverage}% with all tests passing",
|
|
"strategy_actor": "openai/gpt-4",
|
|
"execution_actor": "openai/gpt-4",
|
|
"review_actor": "openai/gpt-4",
|
|
"apply_actor": "openai/gpt-4",
|
|
"estimation_actor": "openai/gpt-4",
|
|
"invariant_actor": "openai/gpt-4",
|
|
"arguments": [
|
|
{
|
|
"name": "target_coverage",
|
|
"type": "integer",
|
|
"required": True,
|
|
"description": "Target coverage percentage",
|
|
"default": 80,
|
|
"min_value": 0,
|
|
"max_value": 100,
|
|
},
|
|
{
|
|
"name": "test_framework",
|
|
"type": "string",
|
|
"required": False,
|
|
"description": "Test framework to use",
|
|
},
|
|
{
|
|
"name": "include_branches",
|
|
"type": "boolean",
|
|
"required": False,
|
|
"description": "Whether to include branch coverage",
|
|
},
|
|
],
|
|
"reusable": True,
|
|
"read_only": False,
|
|
"state": "available",
|
|
"automation_profile": "org/default-profile",
|
|
"invariants": ["No secrets in code", "All tests must pass"],
|
|
"tags": ["coverage", "testing"],
|
|
"created_by": "bench-user",
|
|
}
|
|
|
|
_ARG_MAPPING: dict = {
|
|
"name": "target_coverage",
|
|
"type": "integer",
|
|
"required": True,
|
|
"description": "Target coverage percentage",
|
|
"default": 80,
|
|
"min_value": 0,
|
|
"max_value": 100,
|
|
}
|
|
|
|
|
|
def _make_action() -> Action:
|
|
"""Create a fully-populated Action for benchmarking."""
|
|
return Action.from_config(_FULL_CONFIG)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Benchmark suites
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TimeArgumentParsing:
|
|
"""Benchmark ActionArgument.parse() throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self.simple_arg = "name:string:required:A simple name"
|
|
self.integer_arg = "count:integer:required:Number of items"
|
|
self.optional_arg = "verbose:boolean:optional:Enable verbose output"
|
|
self.no_desc_arg = "limit:float:optional"
|
|
|
|
def time_parse_string_required(self) -> None:
|
|
"""Parse a string/required argument with description."""
|
|
ActionArgument.parse(self.simple_arg)
|
|
|
|
def time_parse_integer_required(self) -> None:
|
|
"""Parse an integer/required argument with description."""
|
|
ActionArgument.parse(self.integer_arg)
|
|
|
|
def time_parse_boolean_optional(self) -> None:
|
|
"""Parse a boolean/optional argument with description."""
|
|
ActionArgument.parse(self.optional_arg)
|
|
|
|
def time_parse_no_description(self) -> None:
|
|
"""Parse an argument without description."""
|
|
ActionArgument.parse(self.no_desc_arg)
|
|
|
|
|
|
class TimeArgumentCoercion:
|
|
"""Benchmark ActionArgument.coerce_value() for each type."""
|
|
|
|
def setup(self) -> None:
|
|
self.int_arg = ActionArgument(
|
|
name="count",
|
|
arg_type=ArgumentType.INTEGER,
|
|
requirement=ArgumentRequirement.REQUIRED,
|
|
description="An integer argument",
|
|
)
|
|
self.float_arg = ActionArgument(
|
|
name="ratio",
|
|
arg_type=ArgumentType.FLOAT,
|
|
requirement=ArgumentRequirement.REQUIRED,
|
|
description="A float argument",
|
|
)
|
|
self.bool_arg = ActionArgument(
|
|
name="verbose",
|
|
arg_type=ArgumentType.BOOLEAN,
|
|
requirement=ArgumentRequirement.OPTIONAL,
|
|
description="A boolean argument",
|
|
)
|
|
self.string_arg = ActionArgument(
|
|
name="label",
|
|
arg_type=ArgumentType.STRING,
|
|
requirement=ArgumentRequirement.REQUIRED,
|
|
description="A string argument",
|
|
)
|
|
self.list_arg = ActionArgument(
|
|
name="items",
|
|
arg_type=ArgumentType.LIST,
|
|
requirement=ArgumentRequirement.OPTIONAL,
|
|
description="A list argument",
|
|
)
|
|
|
|
def time_coerce_integer(self) -> None:
|
|
"""Coerce '42' to integer."""
|
|
self.int_arg.coerce_value("42")
|
|
|
|
def time_coerce_float(self) -> None:
|
|
"""Coerce '3.14' to float."""
|
|
self.float_arg.coerce_value("3.14")
|
|
|
|
def time_coerce_boolean(self) -> None:
|
|
"""Coerce 'true' to boolean."""
|
|
self.bool_arg.coerce_value("true")
|
|
|
|
def time_coerce_string(self) -> None:
|
|
"""Coerce passthrough for string type."""
|
|
self.string_arg.coerce_value("hello world")
|
|
|
|
def time_coerce_list(self) -> None:
|
|
"""Coerce 'a,b,c' to list."""
|
|
self.list_arg.coerce_value("alpha, beta, gamma")
|
|
|
|
|
|
class TimeFromConfig:
|
|
"""Benchmark Action.from_config() with a full config dict."""
|
|
|
|
def setup(self) -> None:
|
|
self.full_config = _FULL_CONFIG.copy()
|
|
self.minimal_config = {
|
|
"name": "local/minimal-action",
|
|
"description": "A minimal action",
|
|
"definition_of_done": "It works",
|
|
"strategy_actor": "openai/gpt-4",
|
|
"execution_actor": "openai/gpt-4",
|
|
}
|
|
|
|
def time_from_config_full(self) -> None:
|
|
"""Construct an Action from a fully-populated config dict."""
|
|
Action.from_config(self.full_config)
|
|
|
|
def time_from_config_minimal(self) -> None:
|
|
"""Construct an Action from a minimal config dict."""
|
|
Action.from_config(self.minimal_config)
|
|
|
|
|
|
class TimeAsCliDict:
|
|
"""Benchmark Action.as_cli_dict() rendering."""
|
|
|
|
def setup(self) -> None:
|
|
self.action = _make_action()
|
|
self.minimal_action = Action(
|
|
namespaced_name=NamespacedName(
|
|
server=None, namespace="local", name="minimal"
|
|
),
|
|
description="Minimal action",
|
|
definition_of_done="Done",
|
|
strategy_actor="openai/gpt-4",
|
|
execution_actor="openai/gpt-4",
|
|
)
|
|
|
|
def time_as_cli_dict_full(self) -> None:
|
|
"""Render as_cli_dict for a fully-populated action."""
|
|
self.action.as_cli_dict()
|
|
|
|
def time_as_cli_dict_minimal(self) -> None:
|
|
"""Render as_cli_dict for a minimal action."""
|
|
self.minimal_action.as_cli_dict()
|
|
|
|
|
|
class TimeTemplateRendering:
|
|
"""Benchmark Action.render_template() with placeholders."""
|
|
|
|
def setup(self) -> None:
|
|
self.action = _make_action()
|
|
self.simple_template = "Coverage target: ${target_coverage}%"
|
|
self.multi_template = (
|
|
"Achieve ${target_coverage}% coverage using ${test_framework} "
|
|
"with branch=${include_branches}"
|
|
)
|
|
self.simple_args = {"target_coverage": 90}
|
|
self.multi_args = {
|
|
"target_coverage": 90,
|
|
"test_framework": "pytest",
|
|
"include_branches": True,
|
|
}
|
|
|
|
def time_render_single_placeholder(self) -> None:
|
|
"""Render a template with a single placeholder."""
|
|
self.action.render_template(self.simple_template, self.simple_args)
|
|
|
|
def time_render_multiple_placeholders(self) -> None:
|
|
"""Render a template with multiple placeholders."""
|
|
self.action.render_template(self.multi_template, self.multi_args)
|
|
|
|
def time_render_description(self) -> None:
|
|
"""Render the action's description template."""
|
|
self.action.render_description(self.simple_args)
|
|
|
|
def time_render_definition_of_done(self) -> None:
|
|
"""Render the action's definition_of_done template."""
|
|
self.action.render_definition_of_done(self.simple_args)
|
|
|
|
|
|
class TimeFromMapping:
|
|
"""Benchmark ActionArgument.from_mapping() from dict."""
|
|
|
|
def setup(self) -> None:
|
|
self.full_mapping = _ARG_MAPPING.copy()
|
|
self.minimal_mapping = {
|
|
"name": "simple_arg",
|
|
"type": "string",
|
|
}
|
|
self.optional_mapping = {
|
|
"name": "verbose",
|
|
"type": "boolean",
|
|
"required": False,
|
|
"description": "Enable verbose output",
|
|
"default": False,
|
|
}
|
|
|
|
def time_from_mapping_full(self) -> None:
|
|
"""Construct ActionArgument from a fully-populated mapping."""
|
|
ActionArgument.from_mapping(self.full_mapping)
|
|
|
|
def time_from_mapping_minimal(self) -> None:
|
|
"""Construct ActionArgument from a minimal mapping."""
|
|
ActionArgument.from_mapping(self.minimal_mapping)
|
|
|
|
def time_from_mapping_optional_with_default(self) -> None:
|
|
"""Construct ActionArgument from an optional mapping with default."""
|
|
ActionArgument.from_mapping(self.optional_mapping)
|