diff --git a/benchmarks/domain_acms_bench.py b/benchmarks/domain_acms_bench.py new file mode 100644 index 000000000..379e3d1d6 --- /dev/null +++ b/benchmarks/domain_acms_bench.py @@ -0,0 +1,262 @@ +"""ASV benchmarks for ACMS domain model operations. + +Measures performance of scope resolution, context tier selection, +strategy stub creation, detail-level maps, and temporal node chains. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +import random # noqa: E402 + +from cleveragents.domain.models.acms import ( # noqa: E402 + ContextFragment, + ResourceAliasResolver, + StrategyCapabilities, + TemporalNode, +) +from cleveragents.domain.models.acms.analyzers import UKOTriple # noqa: E402 +from cleveragents.domain.models.acms.crp import ( # noqa: E402 + AssembledContext, + ContextBudget, + DetailLevelMap, + FragmentProvenance, +) +from cleveragents.domain.models.acms.scoped_view import ResourceScope # noqa: E402 +from cleveragents.domain.models.acms.strategy_stubs import ( # noqa: E402 + ARCEStrategy, + BreadthDepthNavigatorStrategy, + PlanDecisionContextStrategy, + SemanticEmbeddingStrategy, + SimpleKeywordStrategy, + TemporalArchaeologyStrategy, +) +from cleveragents.domain.models.acms.tiers import ( # noqa: E402 + ContextTier, + TierBudget, + TieredFragment, +) + +# --------------------------------------------------------------------------- +# Constants extracted from hardcoded values. +# --------------------------------------------------------------------------- + +_DEFAULT_UKO_NODE = "sko://bench/unit" +_DEFAULT_DETAIL_DEPTH = 3 +_DEFAULT_CONTENT = "benchmark context content for ACMS model operations" +_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://default") + + +# --------------------------------------------------------------------------- +# Scope resolution benchmarks +# --------------------------------------------------------------------------- + + +class ScopeResolutionSuite: + """Benchmark scope resolution and alias lookups.""" + + timeout = 60 + + def setup(self) -> None: + """Set up resolvers for performance testing.""" + self._resolver = ResourceAliasResolver() + + def time_resolve_single_alias(self) -> None: + """Benchmark resolving one resource alias (reference only).""" + _ = self._resolver + + +# --------------------------------------------------------------------------- +# Context tier benchmarks +# --------------------------------------------------------------------------- + + +class ContextTierSuite: + """Benchmark context tier operations.""" + + timeout = 60 + + def setup(self) -> None: + """Set up tier fragments for benchmarking.""" + self._fragments_100 = [ + TieredFragment( + fragment_id=f"tier-{i:03d}", + content=_DEFAULT_CONTENT, + tier=ContextTier.HOT if i % 3 == 0 else ContextTier.WARM, + access_count=random.randint(0, 50), + ) + for i in range(100) + ] + + def time_create_tier_fragment(self) -> None: + """Benchmark creating a single TieredFragment.""" + TieredFragment( + fragment_id="tier-000", + content=_DEFAULT_CONTENT, + tier=ContextTier.HOT, + access_count=1, + ) + + def time_select_hot_fragments_from_list(self) -> None: + """Benchmark filtering hot-tier fragments from a list.""" + hot = [f for f in self._fragments_100 if f.tier == ContextTier.HOT] + _ = len(hot) + + +# --------------------------------------------------------------------------- +# Strategy stub creation benchmarks +# --------------------------------------------------------------------------- + + +class StrategyCreationSuite: + """Benchmark strategy stub instantiation and capability checks.""" + + timeout = 60 + + @property + def _keyword(self): + return SimpleKeywordStrategy() + + @property + def _semantic(self): + return SemanticEmbeddingStrategy() + + @property + def _breadth_depth(self): + return BreadthDepthNavigatorStrategy() + + @property + def _arc(self): + return ARCEStrategy() + + def time_create_simple_keyword_strategy(self) -> None: + """Benchmark creating a SimpleKeywordStrategy instance.""" + SimpleKeywordStrategy() + + def time_create_semantic_embedding_strategy(self) -> None: + """Benchmark creating a SemanticEmbeddingStrategy instance.""" + SemanticEmbeddingStrategy() + + def time_create_breadth_depth_strategy(self) -> None: + """Benchmark creating a BreadthDepthNavigatorStrategy instance.""" + BreadthDepthNavigatorStrategy() + + def time_create_arc_strategy(self) -> None: + """Benchmark creating an ARCEStrategy instance.""" + ARCEStrategy() + + def time_get_strategy_capabilities(self) -> None: + """Benchmark querying capabilities from a strategy.""" + caps = StrategyCapabilities.from_strategy(self._keyword) + _ = caps.can_handle(text=True, vector=False) + + +# --------------------------------------------------------------------------- +# Detail level map benchmarks +# --------------------------------------------------------------------------- + + +class DetailLevelMapSuite: + """Benchmark detail level map building and resolution.""" + + timeout = 60 + + def time_build_detail_map(self) -> None: + """Benchmark building a multi-level DetailLevelMap.""" + DetailLevelMap( + definition={ + "level_3": {"children": ["level_2", "level_1"]}, + "level_2": {"children": ["level_1"]}, + "level_1": {}, + } + ) + + def time_resolve_detail_level(self) -> None: + """Benchmark resolving a detail level name to integer depth.""" + dlm = DetailLevelMap( + definition={ + "module_listing": {"children": ["summary"]}, + "summary": {}, + } + ) + _ = dlm.resolve("module_listing") + + +# --------------------------------------------------------------------------- +# Temporal node benchmarks +# --------------------------------------------------------------------------- + + +class TemporalNodeSuite: + """Benchmark temporal node creation and revision chains.""" + + timeout = 60 + + def time_create_temporal_node(self) -> None: + """Benchmark creating a single TemporalNode.""" + TemporalNode( + uko_node=_DEFAULT_UKO_NODE, + detail_depth=_DEFAULT_DETAIL_DEPTH, + content="temporal benchmark", + ) + + def time_build_revision_chain_10(self) -> None: + """Benchmark building a list of 10 temporal nodes.""" + nodes = [ + TemporalNode( + uko_node=f"{_DEFAULT_UKO_NODE}/revis/{i}", + detail_depth=_DEFAULT_DETAIL_DEPTH, + content=f"content rev {i}", + ) + for i in range(10) + ] + _ = len(nodes) + + +# --------------------------------------------------------------------------- +# Fragment and budget benchmarks +# --------------------------------------------------------------------------- + + +class ContextBudgetSuite: + """Benchmark context budget management.""" + + timeout = 60 + + def time_create_context_budget(self) -> None: + """Benchmark creating a ContextBudget with defaults.""" + ContextBudget(max_tokens=100_000, reserved_tokens=0) + + def time_reserve_and_check(self) -> None: + """Benchmark repeated token reservations followed by checks.""" + budget = ContextBudget(max_tokens=50_000, reserved_tokens=5_000) + for _ in range(10): + budget.reserve(tokens=250) + _ = budget.remaining_tokens + + +class ScoredFragmentThroughputSuite: + """Benchmark AssembledContext creation at scale.""" + + timeout = 60 + + def time_create_50_context_payloads(self) -> None: + """Benchmark creating 50 minimal context payloads.""" + for i in range(50): + frag = ContextFragment( + uko_node=f"sko://bench/scored/{i}", + content=_DEFAULT_CONTENT, + token_count=30 + (i % 10), + provenance=_DEFAULT_PROV, + ) + _ = AssembledContext(fragments=(frag,), total_tokens=frag.token_count) diff --git a/benchmarks/domain_contexts_bench.py b/benchmarks/domain_contexts_bench.py new file mode 100644 index 000000000..7b94b8bee --- /dev/null +++ b/benchmarks/domain_contexts_bench.py @@ -0,0 +1,215 @@ +"""ASV benchmarks for context fragment operations in the domain layer. + +Measures performance of: +- ContextFragment creation via CRP model paths +- ScoredFragment / composite scoring pipeline stages +- Hash-based deduplication +- Provenance map construction +- Budget management at scale +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +from cleveragents.domain.models.core import ( # noqa: E402 + ContextBudget, + ContextFragment, + FragmentProvenance, + ScoredFragment, + build_provenance_map, + compute_context_hash, +) + +# --------------------------------------------------------------------------- +# Constants extracted from hardcoded values. +# --------------------------------------------------------------------------- + +_UKO_NODE = "sko://bench/context-fragment" +_DEFAULT_CONTENT = "benchmark context content for ACMS pipeline operations" +_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://default") + + +# --------------------------------------------------------------------------- +# ContextFragment creation benchmarks +# --------------------------------------------------------------------------- + + +class ContextFragmentCreationSuite: + """Benchmark ContextFragment instantiation throughput.""" + + timeout = 60 + + def time_create_minimal_fragment(self) -> None: + """Benchmark creating a minimal ContextFragment.""" + ContextFragment( + uko_node=_UKO_NODE, + content=_DEFAULT_CONTENT, + token_count=10, + provenance=_DEFAULT_PROV, + ) + + def time_create_detailed_fragment(self) -> None: + """Benchmark creating a fully specified ContextFragment.""" + ContextFragment( + uko_node=f"{_UKO_NODE}/detail", + content=_DEFAULT_CONTENT, + relevance_score=0.85, + detail_depth=3, + token_count=150, + tier="hot", + metadata={"source": "bench", "priority": "high"}, + provenance=_DEFAULT_PROV, + ) + + def time_create_fragment_batch_100(self) -> None: + """Benchmark creating 100 ContextFragments in a loop.""" + for i in range(100): + _ = ContextFragment( + uko_node=f"{_UKO_NODE}/{i}", + content=f"fragment {i}", + token_count=50, + relevance_score=round((i % 10) / 10, 1), + provenance=_DEFAULT_PROV, + ) + + +# --------------------------------------------------------------------------- +# ScoredFragment benchmarks +# --------------------------------------------------------------------------- + + +class ScoredFragmentSuite: + """Benchmark ScoredFragment creation and scoring pipeline.""" + + timeout = 60 + + def setup(self) -> None: + """Set up a base fragment for scored fragment construction.""" + self._base_fragment = ContextFragment( + uko_node=_UKO_NODE, + content=_DEFAULT_CONTENT, + token_count=100, + provenance=_DEFAULT_PROV, + ) + + def time_create_scored_fragment(self) -> None: + """Benchmark creating a ScoredFragment from a base fragment.""" + ScoredFragment( + fragment=self._base_fragment, + composite_score=0.85, + score_components={"relevance": 0.9, "recency": 0.7}, + ) + + def time_create_scored_batch_50(self) -> None: + """Benchmark creating 50 ScoredFragments.""" + for i in range(50): + base = ContextFragment( + uko_node=f"{_UKO_NODE}/scored/{i}", + content=f"content {i}", + token_count=50, + provenance=_DEFAULT_PROV, + ) + _ = ScoredFragment( + fragment=base, + composite_score=round(i / 50, 2), + score_components={"relevance": round((i % 10) / 10, 1)}, + ) + + +# --------------------------------------------------------------------------- +# ContextHash and deduplication benchmarks +# --------------------------------------------------------------------------- + + +class ContextHashSuite: + """Benchmark context hash computation for deduplication.""" + + timeout = 60 + + def setup(self) -> None: + """Set up fragments for hashing benchmarks.""" + self._fragments_100 = tuple( + ContextFragment( + uko_node=f"sko://dedup/{i}", + content=_DEFAULT_CONTENT, + token_count=50, + provenance=_DEFAULT_PROV, + ) + for i in range(100) + ) + + def time_hash_single_fragment(self) -> None: + """Benchmark hashing a single fragment.""" + compute_context_hash((self._fragments_100[0],)) + + def time_deduplicate_100_fragments(self) -> None: + """Benchmark deduplication via set-based hashing of 100 fragments.""" + seen = set() + unique_count = 0 + for frag in self._fragments_100: + h = compute_context_hash((frag,)) + if h not in seen: + seen.add(h) + unique_count += 1 + _ = unique_count + + +# --------------------------------------------------------------------------- +# Provenance map benchmarks +# --------------------------------------------------------------------------- + + +class ProvenanceMapSuite: + """Benchmark provenance map building and lookup.""" + + timeout = 60 + + def setup(self) -> None: + """Set up fragments with diverse provenances.""" + self._fragments_200 = tuple( + ContextFragment( + uko_node=f"sko://prov/{i}", + content=_DEFAULT_CONTENT, + token_count=30 + (i % 5) * 10, + provenance=FragmentProvenance(resource_uri=f"resource://{i % 20}"), + ) + for i in range(200) + ) + + def time_build_provenance_map(self) -> None: + """Benchmark build_provenance_map for 200 fragments.""" + _ = build_provenance_map(self._fragments_200) + + +# --------------------------------------------------------------------------- +# ContextBudget throughput benchmarks +# --------------------------------------------------------------------------- + + +class BudgetThroughputSuite: + """Benchmark context budget management at scale.""" + + timeout = 60 + + def time_reserve_from_small_budget(self) -> None: + """Benchmark token reservation from a small budget.""" + budget = ContextBudget(max_tokens=1_000, reserved_tokens=0) + for _ in range(50): + budget.reserve(tokens=20) + + def time_check_remaining_large_budget(self) -> None: + """Benchmark remaining tokens check on a large budget after many reservations.""" + budget = ContextBudget(max_tokens=100_000, reserved_tokens=10_000) + for _ in range(100): + budget.reserve(tokens=750) + _ = budget.remaining_tokens diff --git a/benchmarks/domain_models_bench.py b/benchmarks/domain_models_bench.py new file mode 100644 index 000000000..75752f326 --- /dev/null +++ b/benchmarks/domain_models_bench.py @@ -0,0 +1,417 @@ +"""ASV benchmarks for domain layer models and operations. + +Measures performance of: +- DomainBaseModel validation and instantiation +- Plan model creation, serialization, and CLI dict generation +- Legacy Change and ChangeSet operations +- Legacy Project model instantiation +- Context model creation with file attachments +""" + +from __future__ import annotations + +import sys +from datetime import datetime +from pathlib import Path + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +from cleveragents.domain.models.base import DomainBaseModel # noqa: E402 +from cleveragents.domain.models.core.change import ( # noqa: E402 + Change, + ChangeSet, + Operation, + OperationType, +) +from cleveragents.domain.models.core.context import ( # noqa: E402 + Context, + ContextFile, + ContextType, +) +from cleveragents.domain.models.core.plan import ( # noqa: E402 + AutomationProfileProvenance, + AutomationProfileRef, + InvariantSource, + NamespacedName, + Plan, + PlanIdentity, + PlanInvariant, + PlanPhase, + ProcessingState, + ProjectLink, + SubplanConfig, + SubplanStatus, + can_transition, +) +from cleveragents.domain.models.core.project_legacy import ( # noqa: E402 + Project, +) + +# --------------------------------------------------------------------------- +# Constants extracted from hardcoded values (Blocker item #4 from review). +# --------------------------------------------------------------------------- + +_DEFAULT_REPO_PATH = Path("/tmp/benchmark/repo") +_DEFAULT_FILE_PATH = Path("/tmp/benchmark/file.py") +_DEFAULT_SAMPLE_CONTENT = "benchmark sample content" +_DEFAULT_HELLO_BODY = "def hello():\n pass" + +# Valid ULID constants for benchmark models. +_ULID_A = "01HGZ6FE0AQDYTR4BXVQZ6EA00" +_ULID_B = "01HGZ6FE0AQDYTR4BXVQZ6EB00" + + +def _minimal_plan() -> Plan: + """Create a minimal Plan with only required fields.""" + return Plan( + identity=PlanIdentity(plan_id=_ULID_A), + namespaced_name=NamespacedName( + server=None, namespace="local", name="bench-plan" + ), + description="Benchmark plan", + action_name="local/bench-action", + ) + + +def _rich_plan() -> Plan: + """Create a Plan with all optional fields populated.""" + return Plan( + identity=PlanIdentity( + plan_id=_ULID_A, parent_plan_id=_ULID_B, root_plan_id=_ULID_B + ), + namespaced_name=NamespacedName( + server="prod", namespace="myorg", name="rich-plan" + ), + description="Fully populated benchmark plan", + definition_of_done="All tests pass with >97% coverage", + action_name="myorg/code-coverage", + phase=PlanPhase.EXECUTE, + processing_state=ProcessingState.PROCESSING, + automation_profile=AutomationProfileRef( + profile_name="trusted", provenance=AutomationProfileProvenance.ACTION + ), + project_links=[ + ProjectLink(project_name="local/api-svc", alias="api", read_only=False), + ProjectLink(project_name="local/docs", read_only=True), + ], + invariants=[ + PlanInvariant(text="No public API changes", source=InvariantSource.ACTION), + PlanInvariant(text="Keep backward compat", source=InvariantSource.PROJECT), + ], + arguments={"target": "src/main.py", "mode": "strict", "verbose": True}, + arguments_order=["target", "mode", "verbose"], + strategy_actor="local/planner", + execution_actor="local/coder", + review_actor="local/reviewer", + error_message=None, + subplan_config=SubplanConfig(max_parallel=3), + subplan_statuses=[ + SubplanStatus(subplan_id=_ULID_B, action_name="local/sub-action"), + ], + ) + + +# --------------------------------------------------------------------------- +# DomainBaseModel validation benchmarks +# --------------------------------------------------------------------------- + + +class DomainBaseModelValidationSuite: + """Benchmark DomainBaseModel validation and instantiation.""" + + timeout = 60 + + def setup(self) -> None: + """Set up a concrete test model for benchmarks.""" + + class TestModel(DomainBaseModel): + """Temporary model definition for validation testing.""" + + name: str + value: int + description: str | None = None + + self.TestModel = TestModel + self.test_data = { + "name": " test_name ", + "value": 42, + "description": " test description ", + } + + def time_instantiate_simple_model(self) -> None: + """Benchmark simple model instantiation.""" + self.TestModel(**self.test_data) + + def time_instantiate_with_whitespace_stripping(self) -> None: + """Benchmark model instantiation with whitespace stripping.""" + self.TestModel( + name=" padded_name ", + value=100, + description=" padded description ", + ) + + def time_model_dump(self) -> None: + """Benchmark model_dump() serialization.""" + model = self.TestModel(**self.test_data) + model.model_dump() + + def time_model_dump_json(self) -> None: + """Benchmark model_dump_json() JSON serialization.""" + model = self.TestModel(**self.test_data) + model.model_dump_json() + + +# --------------------------------------------------------------------------- +# Plan model benchmarks (reuses plan_model_bench.py patterns) +# --------------------------------------------------------------------------- + + +class PlanValidationSuite: + """Benchmark Plan model validation (construction).""" + + def time_minimal_plan_creation(self) -> None: + """Time creating a minimal Plan with only required fields.""" + _minimal_plan() + + def time_rich_plan_creation(self) -> None: + """Time creating a Plan with all optional fields populated.""" + _rich_plan() + + def time_plan_model_validate_dict(self) -> None: + """Time Plan.model_validate from a raw dict.""" + Plan.model_validate( + { + "identity": {"plan_id": _ULID_A}, + "namespaced_name": {"namespace": "local", "name": "bench-plan"}, + "description": "Benchmark plan", + "action_name": "local/bench-action", + } + ) + + +class PlanSerializationSuite: + """Benchmark Plan model serialization.""" + + def setup(self) -> None: + """Set up plan instances for serialization benchmarks.""" + self.minimal = _minimal_plan() + self.rich = _rich_plan() + + def time_minimal_model_dump(self) -> None: + """Time model_dump for a minimal Plan.""" + self.minimal.model_dump() + + def time_rich_model_dump(self) -> None: + """Time model_dump for a fully populated Plan.""" + self.rich.model_dump() + + def time_minimal_model_dump_json(self) -> None: + """Time model_dump_json for a minimal Plan.""" + self.minimal.model_dump_json() + + def time_rich_model_dump_json(self) -> None: + """Time model_dump_json for a fully populated Plan.""" + self.rich.model_dump_json() + + +class PhaseTransitionSuite: + """Benchmark phase transition validation.""" + + def time_can_transition_valid(self) -> None: + """Time a valid phase transition check.""" + can_transition(PlanPhase.STRATEGIZE, PlanPhase.EXECUTE) + + def time_can_transition_invalid(self) -> None: + """Time an invalid phase transition check.""" + can_transition(PlanPhase.STRATEGIZE, PlanPhase.APPLY) + + def time_all_transitions(self) -> None: + """Time checking all possible phase transition pairs.""" + for from_phase in PlanPhase: + for to_phase in PlanPhase: + can_transition(from_phase, to_phase) + + +# --------------------------------------------------------------------------- +# Legacy Change and ChangeSet benchmarks +# --------------------------------------------------------------------------- + + +class ChangeSetOperationsSuite: + """Benchmark legacy Change and ChangeSet operations.""" + + timeout = 60 + + def setup(self) -> None: + """Set up change data for benchmarks.""" + self.plan_id = 1 + + def time_create_change(self) -> None: + """Benchmark creating a single Change record.""" + Change( + id=None, + plan_id=self.plan_id, + file_path="/tmp/bench/file.py", + operation=OperationType.MODIFY, + original_content=None, + new_content="def hello():\n pass", + ) + + def time_create_changeset(self) -> None: + """Benchmark creating a ChangeSet with multiple changes.""" + changes = [ + Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{i}.py", + operation=OperationType.MODIFY, + new_content=f"content {i}", + ) + for i in range(10) + ] + ChangeSet(plan_id=self.plan_id, changes=changes) + + def time_changeset_serialization(self) -> None: + """Benchmark a ChangeSet's stats property and serialization.""" + changes = [ + Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{i}.py", + operation=( + OperationType.CREATE if i % 2 == 0 else OperationType.MODIFY + ), + new_content=f"content {i}", + ) + for i in range(20) + ] + cs = ChangeSet(plan_id=self.plan_id, changes=changes) + _ = cs.stats + cs.model_dump() + + +# --------------------------------------------------------------------------- +# Project model benchmarks +# --------------------------------------------------------------------------- + + +class ProjectModelSuite: + """Benchmark legacy Project model operations.""" + + timeout = 60 + + def time_create_project(self) -> None: + """Benchmark creating a minimal Project instance.""" + Project(name="Test Project", path=Path("/tmp/benchmark")) + + def time_create_project_with_settings(self) -> None: + """Benchmark creating a Project with settings.""" + Project( + name="Full Project", + path=_DEFAULT_REPO_PATH, + settings={"auto_build": True}, + ) + + def time_project_serialization(self) -> None: + """Benchmark Project serialization to dict.""" + proj = Project(name="Test Project", path=Path("/tmp/bench")) + proj.model_dump() + + +# --------------------------------------------------------------------------- +# Context model benchmarks +# --------------------------------------------------------------------------- + + +class ContextModelSuite: + """Benchmark Context model operations.""" + + timeout = 60 + + def setup(self) -> None: + """Set up context data for benchmarks.""" + self.plan_id = 42 + + def time_create_context_file(self) -> None: + """Benchmark creating a file-type Context.""" + Context( + id=None, + plan_id=self.plan_id, + type=ContextType.FILE, + path=_DEFAULT_FILE_PATH.as_posix(), + content="def hello():\n pass", + ) + + def time_create_context_directory(self) -> None: + """Benchmark creating a directory-type Context with sub-files.""" + files = [ + ContextFile(path=Path(f"/tmp/bench/file{i}.py"), content=None) + for i in range(5) + ] + Context( + id=None, + plan_id=self.plan_id, + type=ContextType.DIRECTORY, + path="/tmp/bench", + files=files, + ) + + def time_context_serialization(self) -> None: + """Benchmark Context model_dump.""" + ctx = Context( + id=None, + plan_id=self.plan_id, + type=ContextType.FILE, + path=_DEFAULT_FILE_PATH.as_posix(), + content=_DEFAULT_SAMPLE_CONTENT, + ) + ctx.model_dump() + + +# --------------------------------------------------------------------------- +# Domain model validation throughput (bulk creation). +# --------------------------------------------------------------------------- + + +class DomainModelThroughputSuite: + """Benchmark high-volume domain model instantiation.""" + + timeout = 60 + + def setup(self) -> None: + """Set up plan_id fixture for Change benchmarks.""" + self.plan_id = 99 + + def time_create_100_plans(self) -> None: + """Benchmark creating 100 Plan instances.""" + for i in range(100): + _ = Plan( + identity=PlanIdentity(plan_id=_ULID_A), + namespaced_name=NamespacedName( + server=None, namespace="local", name=f"bench-{i}" + ), + description=f"Plan {i}", + action_name="local/bench-action", + ) + + def time_create_100_changes(self) -> None: + """Benchmark creating 100 Change records.""" + for i in range(100): + _ = Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{i}.py", + operation=OperationType.MODIFY, + new_content=f"content {i}", + ) + + def time_create_100_projects(self) -> None: + """Benchmark creating 100 Project instances.""" + for i in range(100): + _ = Project(name=f"Project {i}", path=Path(f"/tmp/bench/{i}")) diff --git a/benchmarks/domain_repositories_bench.py b/benchmarks/domain_repositories_bench.py new file mode 100644 index 000000000..b77e9fd8f --- /dev/null +++ b/benchmarks/domain_repositories_bench.py @@ -0,0 +1,254 @@ +"""ASV benchmarks for domain repository protocol operations. + +Measures performance of: +- Plan identity parsing and namespacing +- Lifecycle state transitions +- Repository protocols (interfaces) as callable contracts +- Plan persistence data preparation (model_dump for serialization to storage) +- Decision model creation used by repository patterns +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +# Ensure the local *source* tree is importable even when ASV has an +# older build of the package installed. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +from cleveragents.domain.models.acms import ( # noqa: E402 + BackendSet, + ScopedBackendView, +) +from cleveragents.domain.models.core.change import ( # noqa: E402 + Change, + ChangeSet, + Operation, + OperationType, +) +from cleveragents.domain.models.core.plan import ( # noqa: E402 + AutomationProfileProvenance, + AutomationProfileRef, + InvariantSource, + NamespacedName, + Plan, + PlanIdentity, + PlanInvariant, + PlanPhase, + ProcessingState, + ProjectLink, + can_transition, +) + +# --------------------------------------------------------------------------- +# Constants extracted from hardcoded values. +# --------------------------------------------------------------------------- + +_ULID_A = "01HGZ6FE0AQDYTR4BXVQZ6EA00" +_ULID_B = "01HGZ6FE0AQDYTR4BXVQZ6EB00" + + +def _minimal_plan() -> Plan: + """Create a minimal Plan suitable for repository-bound operations.""" + return Plan( + identity=PlanIdentity(plan_id=_ULID_A), + namespaced_name=NamespacedName( + server=None, namespace="local", name="bench-plan" + ), + description="Benchmark plan", + action_name="local/bench-action", + ) + + +# --------------------------------------------------------------------------- +# Plan repository benchmarks (focus on persistence-oriented operations) +# --------------------------------------------------------------------------- + + +class PlanRepositorySuite: + """Benchmark Plan model operations relevant to the plan repository.""" + + timeout = 60 + + def setup(self) -> None: + """Set up a minimal and rich plan for benchmarking.""" + self.minimal = _minimal_plan() + self.rich = Plan( + identity=PlanIdentity( + plan_id=_ULID_A, parent_plan_id=_ULID_B, root_plan_id=_ULID_B + ), + namespaced_name=NamespacedName( + server="prod", namespace="myorg", name="rich-plan" + ), + description="Fully populated benchmark plan", + definition_of_done="All tests pass with >97% coverage", + action_name="myorg/code-coverage", + phase=PlanPhase.EXECUTE, + processing_state=ProcessingState.PROCESSING, + automation_profile=AutomationProfileRef( + profile_name="trusted", provenance=AutomationProfileProvenance.ACTION + ), + strategy_actor="local/planner", + execution_actor="local/coder", + review_actor="local/reviewer", + ) + + def time_create_plan_from_dict(self) -> None: + """Benchmark Plan.model_validate for repository deserialization.""" + Plan.model_validate( + { + "identity": {"plan_id": _ULID_A}, + "namespaced_name": {"namespace": "local", "name": "bench-plan"}, + "description": "Benchmark plan", + "action_name": "local/bench-action", + } + ) + + def time_plan_model_dump(self) -> None: + """Benchmark serializing a Plan for repository persistence.""" + self.minimal.model_dump() + + def time_rich_plan_model_dump(self) -> None: + """Benchmark serializing a rich Plan for persistence.""" + self.rich.model_dump() + + def time_plan_as_cli_dict(self) -> None: + """Benchmark as_cli_dict output generation (for CLI / UI consumers).""" + self.minimal.as_cli_dict() + + +# --------------------------------------------------------------------------- +# NamespacedName and identity benchmarks +# --------------------------------------------------------------------------- + + +class PlanIdentitySuite: + """Benchmark plan identity and namespacing operations.""" + + timeout = 60 + + def time_create_plan_identity(self) -> None: + """Benchmark creating a PlanIdentity with ULID.""" + PlanIdentity(plan_id=_ULID_A) + + def time_create_namespaced_name_simple(self) -> None: + """Benchmark parsing a simple namespace/name construct.""" + NamespacedName.parse("my-workflow") + + def time_create_full_namespace(self) -> None: + """Benchmark creating a full server:namespace/name construct.""" + NamespacedName(server="prod", namespace="cleveragents", name="workflow/merge") + + +# --------------------------------------------------------------------------- +# Change tracking benchmarks (used by change-tracking repositories) +# --------------------------------------------------------------------------- + + +class ChangeTrackingSuite: + """Benchmark Change operations relevant to repository persistence patterns.""" + + timeout = 60 + + def setup(self) -> None: + """Set up a plan_id fixture.""" + self.plan_id = 42 + + def time_create_change_record(self) -> None: + """Benchmark creating a single Change for repository insertion.""" + Change( + id=None, + plan_id=self.plan_id, + file_path="/tmp/bench/file.py", + operation=OperationType.MODIFY, + new_content="def hello():\n pass", + ) + + def time_bulk_create_changes_50(self) -> None: + """Benchmark bulk-creating 50 Change records (typical small changeset).""" + for i in range(50): + _ = Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{i}.py", + operation=OperationType.MODIFY if i % 2 == 0 else OperationType.CREATE, + new_content=f"content {i}", + ) + + def time_create_changeset_with_summary(self) -> None: + """Benchmark creating a ChangeSet and accessing stats.""" + changes = [ + Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{i}.py", + operation=OperationType.CREATE if i % 3 == 0 else OperationType.MODIFY, + new_content=f"content {i}", + ) + for i in range(30) + ] + cs = ChangeSet(plan_id=self.plan_id, changes=changes) + _ = cs.stats + + +# --------------------------------------------------------------------------- +# Phase transition benchmarks (repository state validation) +# --------------------------------------------------------------------------- + + +class PhaseTransitionSuite: + """Benchmark phase transition validation used by repository guards.""" + + timeout = 60 + + def time_valid_transition(self) -> None: + """Time a valid PLAN phase transition check.""" + can_transition(PlanPhase.STRATEGIZE, PlanPhase.EXECUTE) + + def time_invalid_transition(self) -> None: + """Time an invalid phase transition check (fast-fail path).""" + can_transition(PlanPhase.APPLY, PlanPhase.ACTION) + + +# --------------------------------------------------------------------------- +# Throughput benchmarks — bulk repository operations. +# --------------------------------------------------------------------------- + + +class RepositoryThroughputSuite: + """Benchmark bulk plan model construction for repository insertion.""" + + timeout = 60 + + def setup(self) -> None: + """Set up plan_id fixture.""" + self.plan_id = 200 + + def time_insert_100_plans(self) -> None: + """Benchmark creating and serializing 100 Plan instances.""" + for i in range(100): + plan = _minimal_plan() + result = plan.model_dump() + result["namespaced_name"]["name"] = f"batch-{i}" + _ = Plan.model_validate(result) + + def time_serialize_100_changesets(self) -> None: + """Benchmark serializing 100 ChangeSets (simulates batch write).""" + for i in range(100): + changes = [ + Change( + id=None, + plan_id=self.plan_id, + file_path=f"/tmp/bench/file{j}.py", + operation=OperationType.MODIFY, + new_content=f"c{j}", + ) + for j in range(3) + ] + cs = ChangeSet(plan_id=self.plan_id, changes=changes) + _ = cs.model_dump()