"""ASV benchmarks for Action domain model operations. Measures the performance of: - ActionArgument.parse() from colon-delimited strings - ActionArgument.coerce_value() type coercion - Action.from_config() construction from YAML config dicts - Action.as_cli_dict() stable CLI rendering - Action.render_template() placeholder substitution - ActionArgument.from_mapping() construction from dicts """ from __future__ import annotations import sys from pathlib import Path try: from cleveragents.domain.models.core.action import ( Action, ActionArgument, ArgumentRequirement, ArgumentType, ) from cleveragents.domain.models.core.plan import NamespacedName except ModuleNotFoundError: sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src")) from cleveragents.domain.models.core.action import ( Action, ActionArgument, ArgumentRequirement, ArgumentType, ) from cleveragents.domain.models.core.plan import NamespacedName # --------------------------------------------------------------------------- # Shared fixtures # --------------------------------------------------------------------------- _FULL_CONFIG: dict = { "name": "local/code-coverage", "description": "Increase test coverage to ${target_coverage}%", "long_description": "Systematically improve test coverage across the project.", "definition_of_done": "Coverage reaches ${target_coverage}% with all tests passing", "strategy_actor": "openai/gpt-4", "execution_actor": "openai/gpt-4", "review_actor": "openai/gpt-4", "apply_actor": "openai/gpt-4", "estimation_actor": "openai/gpt-4", "invariant_actor": "openai/gpt-4", "arguments": [ { "name": "target_coverage", "type": "integer", "required": True, "description": "Target coverage percentage", "default": 80, "min_value": 0, "max_value": 100, }, { "name": "test_framework", "type": "string", "required": False, "description": "Test framework to use", }, { "name": "include_branches", "type": "boolean", "required": False, "description": "Whether to include branch coverage", }, ], "reusable": True, "read_only": False, "state": "available", "automation_profile": "org/default-profile", "invariants": ["No secrets in code", "All tests must pass"], "tags": ["coverage", "testing"], "created_by": "bench-user", } _ARG_MAPPING: dict = { "name": "target_coverage", "type": "integer", "required": True, "description": "Target coverage percentage", "default": 80, "min_value": 0, "max_value": 100, } def _make_action() -> Action: """Create a fully-populated Action for benchmarking.""" return Action.from_config(_FULL_CONFIG) # --------------------------------------------------------------------------- # Benchmark suites # --------------------------------------------------------------------------- class TimeArgumentParsing: """Benchmark ActionArgument.parse() throughput.""" def setup(self) -> None: self.simple_arg = "name:string:required:A simple name" self.integer_arg = "count:integer:required:Number of items" self.optional_arg = "verbose:boolean:optional:Enable verbose output" self.no_desc_arg = "limit:float:optional" def time_parse_string_required(self) -> None: """Parse a string/required argument with description.""" ActionArgument.parse(self.simple_arg) def time_parse_integer_required(self) -> None: """Parse an integer/required argument with description.""" ActionArgument.parse(self.integer_arg) def time_parse_boolean_optional(self) -> None: """Parse a boolean/optional argument with description.""" ActionArgument.parse(self.optional_arg) def time_parse_no_description(self) -> None: """Parse an argument without description.""" ActionArgument.parse(self.no_desc_arg) class TimeArgumentCoercion: """Benchmark ActionArgument.coerce_value() for each type.""" def setup(self) -> None: self.int_arg = ActionArgument( name="count", arg_type=ArgumentType.INTEGER, requirement=ArgumentRequirement.REQUIRED, description="An integer argument", ) self.float_arg = ActionArgument( name="ratio", arg_type=ArgumentType.FLOAT, requirement=ArgumentRequirement.REQUIRED, description="A float argument", ) self.bool_arg = ActionArgument( name="verbose", arg_type=ArgumentType.BOOLEAN, requirement=ArgumentRequirement.OPTIONAL, description="A boolean argument", ) self.string_arg = ActionArgument( name="label", arg_type=ArgumentType.STRING, requirement=ArgumentRequirement.REQUIRED, description="A string argument", ) self.list_arg = ActionArgument( name="items", arg_type=ArgumentType.LIST, requirement=ArgumentRequirement.OPTIONAL, description="A list argument", ) def time_coerce_integer(self) -> None: """Coerce '42' to integer.""" self.int_arg.coerce_value("42") def time_coerce_float(self) -> None: """Coerce '3.14' to float.""" self.float_arg.coerce_value("3.14") def time_coerce_boolean(self) -> None: """Coerce 'true' to boolean.""" self.bool_arg.coerce_value("true") def time_coerce_string(self) -> None: """Coerce passthrough for string type.""" self.string_arg.coerce_value("hello world") def time_coerce_list(self) -> None: """Coerce 'a,b,c' to list.""" self.list_arg.coerce_value("alpha, beta, gamma") class TimeFromConfig: """Benchmark Action.from_config() with a full config dict.""" def setup(self) -> None: self.full_config = _FULL_CONFIG.copy() self.minimal_config = { "name": "local/minimal-action", "description": "A minimal action", "definition_of_done": "It works", "strategy_actor": "openai/gpt-4", "execution_actor": "openai/gpt-4", } def time_from_config_full(self) -> None: """Construct an Action from a fully-populated config dict.""" Action.from_config(self.full_config) def time_from_config_minimal(self) -> None: """Construct an Action from a minimal config dict.""" Action.from_config(self.minimal_config) class TimeAsCliDict: """Benchmark Action.as_cli_dict() rendering.""" def setup(self) -> None: self.action = _make_action() self.minimal_action = Action( namespaced_name=NamespacedName( server=None, namespace="local", name="minimal" ), description="Minimal action", definition_of_done="Done", strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", ) def time_as_cli_dict_full(self) -> None: """Render as_cli_dict for a fully-populated action.""" self.action.as_cli_dict() def time_as_cli_dict_minimal(self) -> None: """Render as_cli_dict for a minimal action.""" self.minimal_action.as_cli_dict() class TimeTemplateRendering: """Benchmark Action.render_template() with placeholders.""" def setup(self) -> None: self.action = _make_action() self.simple_template = "Coverage target: ${target_coverage}%" self.multi_template = ( "Achieve ${target_coverage}% coverage using ${test_framework} " "with branch=${include_branches}" ) self.simple_args = {"target_coverage": 90} self.multi_args = { "target_coverage": 90, "test_framework": "pytest", "include_branches": True, } def time_render_single_placeholder(self) -> None: """Render a template with a single placeholder.""" self.action.render_template(self.simple_template, self.simple_args) def time_render_multiple_placeholders(self) -> None: """Render a template with multiple placeholders.""" self.action.render_template(self.multi_template, self.multi_args) def time_render_description(self) -> None: """Render the action's description template.""" self.action.render_description(self.simple_args) def time_render_definition_of_done(self) -> None: """Render the action's definition_of_done template.""" self.action.render_definition_of_done(self.simple_args) class TimeFromMapping: """Benchmark ActionArgument.from_mapping() from dict.""" def setup(self) -> None: self.full_mapping = _ARG_MAPPING.copy() self.minimal_mapping = { "name": "simple_arg", "type": "string", } self.optional_mapping = { "name": "verbose", "type": "boolean", "required": False, "description": "Enable verbose output", "default": False, } def time_from_mapping_full(self) -> None: """Construct ActionArgument from a fully-populated mapping.""" ActionArgument.from_mapping(self.full_mapping) def time_from_mapping_minimal(self) -> None: """Construct ActionArgument from a minimal mapping.""" ActionArgument.from_mapping(self.minimal_mapping) def time_from_mapping_optional_with_default(self) -> None: """Construct ActionArgument from an optional mapping with default.""" ActionArgument.from_mapping(self.optional_mapping)