diff --git a/benchmarks/cli_format_bench.py b/benchmarks/cli_format_bench.py index 246f3d66b..6ba70958c 100644 --- a/benchmarks/cli_format_bench.py +++ b/benchmarks/cli_format_bench.py @@ -83,10 +83,10 @@ def _mock_plan(name: str = "local/bench-plan") -> Plan: class ActionListFormatSuite: """Benchmark action list --format throughput.""" - params: list[str] = ["json", "yaml", "plain", "table", "rich"] - param_names: list[str] = ["format"] + params: list[str] = ["json", "yaml", "plain", "table", "rich"] # noqa: RUF012 + param_names: list[str] = ["format"] # noqa: RUF012 - def setup(self) -> None: + def setup(self, fmt: str) -> None: self._svc = MagicMock() self._svc.list_actions.return_value = [ _mock_action(f"local/action-{i}") for i in range(20) @@ -97,7 +97,7 @@ class ActionListFormatSuite: ) self._patcher.start() - def teardown(self) -> None: + def teardown(self, fmt: str) -> None: self._patcher.stop() def time_list(self, fmt: str) -> None: @@ -107,10 +107,10 @@ class ActionListFormatSuite: class PlanStatusFormatSuite: """Benchmark plan status --format throughput.""" - params: list[str] = ["json", "yaml", "plain", "table", "rich"] - param_names: list[str] = ["format"] + params: list[str] = ["json", "yaml", "plain", "table", "rich"] # noqa: RUF012 + param_names: list[str] = ["format"] # noqa: RUF012 - def setup(self) -> None: + def setup(self, fmt: str) -> None: self._svc = MagicMock() self._svc.get_plan.return_value = _mock_plan() self._patcher = patch( @@ -119,7 +119,7 @@ class PlanStatusFormatSuite: ) self._patcher.start() - def teardown(self) -> None: + def teardown(self, fmt: str) -> None: self._patcher.stop() def time_status(self, fmt: str) -> None: diff --git a/benchmarks/plan_lifecycle_persistence_bench.py b/benchmarks/plan_lifecycle_persistence_bench.py index 807232a80..c17f2db76 100644 --- a/benchmarks/plan_lifecycle_persistence_bench.py +++ b/benchmarks/plan_lifecycle_persistence_bench.py @@ -117,6 +117,7 @@ class TimePhaseTransitionPersisted: timeout = 30 def setup(self) -> None: + self._counter = 0 self.uow, _ = _build_uow() self.settings = Settings() self.svc = PlanLifecycleService(settings=self.settings, unit_of_work=self.uow) @@ -127,10 +128,13 @@ class TimePhaseTransitionPersisted: strategy_actor="openai/gpt-4", execution_actor="openai/gpt-4", ) - plan = self.svc.use_action(action_name="local/bench-transition-action") - self.plan_id = plan.identity.plan_id - self.svc.start_strategize(self.plan_id) - self.svc.complete_strategize(self.plan_id) def time_execute_transition(self) -> None: - self.svc.execute_plan(self.plan_id) + self._counter += 1 + plan = self.svc.use_action( + action_name="local/bench-transition-action", + ) + plan_id = plan.identity.plan_id + self.svc.start_strategize(plan_id) + self.svc.complete_strategize(plan_id) + self.svc.execute_plan(plan_id) diff --git a/benchmarks/plan_model_bench.py b/benchmarks/plan_model_bench.py index 4a6cb5359..4a7c4fa12 100644 --- a/benchmarks/plan_model_bench.py +++ b/benchmarks/plan_model_bench.py @@ -152,7 +152,7 @@ class PhaseTransitionSuite: def time_can_transition_invalid(self) -> None: """Time an invalid phase transition check.""" - can_transition(PlanPhase.STRATEGIZE, PlanPhase.APPLIED) + can_transition(PlanPhase.STRATEGIZE, PlanPhase.APPLY) def time_all_transitions(self) -> None: """Time checking all possible phase transition pairs.""" diff --git a/benchmarks/plan_phase_migration_bench.py b/benchmarks/plan_phase_migration_bench.py index d6c4543a3..8bf95f5c7 100644 --- a/benchmarks/plan_phase_migration_bench.py +++ b/benchmarks/plan_phase_migration_bench.py @@ -41,6 +41,7 @@ class PlanInsertActionPhase: """Benchmark inserting plans with phase='action' (new phase).""" def setup(self) -> None: + self._batch = 0 self.engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(self.engine) self.session_factory = sessionmaker(bind=self.engine) @@ -67,17 +68,19 @@ class PlanInsertActionPhase: self.engine.dispose() def time_insert_100_action_phase_plans(self) -> None: + offset = self._batch * 100 + self._batch += 1 session = self.session_factory() now = _now_iso() for i in range(100): plan = LifecyclePlanModel() - plan.plan_id = _make_ulid(i) + plan.plan_id = _make_ulid(offset + i) plan.action_name = "bench/phase-action" - plan.namespaced_name = f"bench/plan-{i}" + plan.namespaced_name = f"bench/plan-{offset + i}" plan.namespace = "bench" plan.phase = "action" plan.processing_state = "queued" - plan.description = f"Benchmark plan {i}" + plan.description = f"Benchmark plan {offset + i}" plan.tags_json = "[]" plan.created_at = now plan.updated_at = now @@ -90,6 +93,8 @@ class PlanInsertApplyTerminalStates: """Benchmark inserting plans with Apply terminal states.""" def setup(self) -> None: + self._applied_batch = 0 + self._constrained_batch = 0 self.engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(self.engine) self.session_factory = sessionmaker(bind=self.engine) @@ -115,17 +120,20 @@ class PlanInsertApplyTerminalStates: self.engine.dispose() def time_insert_50_applied_plans(self) -> None: + offset = self._applied_batch * 50 + self._applied_batch += 1 session = self.session_factory() now = _now_iso() for i in range(50): + uid = offset + i plan = LifecyclePlanModel() - plan.plan_id = _make_ulid(1000 + i) + plan.plan_id = _make_ulid(100_000 + uid) plan.action_name = "bench/terminal-action" - plan.namespaced_name = f"bench/applied-{i}" + plan.namespaced_name = f"bench/applied-{uid}" plan.namespace = "bench" plan.phase = "apply" plan.processing_state = "applied" - plan.description = f"Applied plan {i}" + plan.description = f"Applied plan {uid}" plan.tags_json = "[]" plan.created_at = now plan.updated_at = now @@ -134,17 +142,20 @@ class PlanInsertApplyTerminalStates: session.close() def time_insert_50_constrained_plans(self) -> None: + offset = self._constrained_batch * 50 + self._constrained_batch += 1 session = self.session_factory() now = _now_iso() for i in range(50): + uid = offset + i plan = LifecyclePlanModel() - plan.plan_id = _make_ulid(2000 + i) + plan.plan_id = _make_ulid(200_000 + uid) plan.action_name = "bench/terminal-action" - plan.namespaced_name = f"bench/constrained-{i}" + plan.namespaced_name = f"bench/constrained-{uid}" plan.namespace = "bench" plan.phase = "apply" plan.processing_state = "constrained" - plan.description = f"Constrained plan {i}" + plan.description = f"Constrained plan {uid}" plan.tags_json = "[]" plan.created_at = now plan.updated_at = now diff --git a/benchmarks/project_migration_bench.py b/benchmarks/project_migration_bench.py index 4a561d01b..eb0892925 100644 --- a/benchmarks/project_migration_bench.py +++ b/benchmarks/project_migration_bench.py @@ -42,6 +42,8 @@ class ProjectInsert: """Benchmark inserting project records.""" def setup(self) -> None: + self._single_ctr = 0 + self._batch_ctr = 0 self.engine = create_engine("sqlite:///:memory:") event.listen(self.engine, "connect", _set_sqlite_pragma) with self.engine.connect() as conn: @@ -54,9 +56,10 @@ class ProjectInsert: self.engine.dispose() def time_insert_single_project(self) -> None: + self._single_ctr += 1 session = self.session_factory() proj = NamespacedProjectModel( - namespaced_name="bench/single-project", + namespaced_name=f"bench/single-project-{self._single_ctr}", namespace="bench", tags_json="[]", created_at=_now_iso(), @@ -67,13 +70,16 @@ class ProjectInsert: session.close() def time_insert_100_projects(self) -> None: + offset = self._batch_ctr * 100 + self._batch_ctr += 1 session = self.session_factory() for i in range(100): + uid = offset + i proj = NamespacedProjectModel( - namespaced_name=f"bench/project-{i}", + namespaced_name=f"bench/project-{uid}", namespace="bench", - description=f"Project {i}", - tags_json=json.dumps([f"tag-{i}"]), + description=f"Project {uid}", + tags_json=json.dumps([f"tag-{uid}"]), created_at=_now_iso(), updated_at=_now_iso(), ) @@ -86,6 +92,7 @@ class ProjectLinkInsert: """Benchmark inserting project resource link records.""" def setup(self) -> None: + self._batch_ctr = 0 self.engine = create_engine("sqlite:///:memory:") event.listen(self.engine, "connect", _set_sqlite_pragma) with self.engine.connect() as conn: @@ -115,8 +122,8 @@ class ProjectLinkInsert: ) session.add(proj) - # Create 50 resources - for i in range(50): + # Create resources (large pool so repeated iterations can use unique ones) + for i in range(5000): r = ResourceModel() r.resource_id = _make_ulid(i) r.type_name = "bench/git-checkout" @@ -132,12 +139,15 @@ class ProjectLinkInsert: self.engine.dispose() def time_insert_50_links(self) -> None: + offset = self._batch_ctr * 50 + self._batch_ctr += 1 session = self.session_factory() for i in range(50): + uid = offset + i link = ProjectResourceLinkModel( - link_id=_make_ulid(1000 + i), + link_id=_make_ulid(10_000 + uid), project_name="bench/link-project", - resource_id=_make_ulid(i), + resource_id=_make_ulid(uid), read_only=i % 2 == 0, created_at=_now_iso(), ) diff --git a/benchmarks/resource_registry_migration_bench.py b/benchmarks/resource_registry_migration_bench.py index 38cbf41fd..8507b323b 100644 --- a/benchmarks/resource_registry_migration_bench.py +++ b/benchmarks/resource_registry_migration_bench.py @@ -45,6 +45,8 @@ class ResourceTypeInsert: """Benchmark inserting resource type records.""" def setup(self) -> None: + self._single_ctr = 0 + self._batch_ctr = 0 self.engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(self.engine) self.session_factory = sessionmaker(bind=self.engine) @@ -53,9 +55,10 @@ class ResourceTypeInsert: self.engine.dispose() def time_insert_single_resource_type(self) -> None: + self._single_ctr += 1 session = self.session_factory() rt = ResourceTypeModel() - rt.name = "bench/type-single" + rt.name = f"bench/type-single-{self._single_ctr}" rt.namespace = "bench" rt.resource_kind = "physical" rt.user_addable = True @@ -66,14 +69,17 @@ class ResourceTypeInsert: session.close() def time_insert_100_resource_types(self) -> None: + offset = self._batch_ctr * 100 + self._batch_ctr += 1 session = self.session_factory() for i in range(100): + uid = offset + i rt = ResourceTypeModel() - rt.name = f"bench/type-{i}" + rt.name = f"bench/type-{uid}" rt.namespace = "bench" rt.resource_kind = "physical" if i % 2 == 0 else "virtual" rt.user_addable = i % 3 == 0 - rt.args_schema_json = json.dumps([{"flag": f"--arg-{i}"}]) + rt.args_schema_json = json.dumps([{"flag": f"--arg-{uid}"}]) rt.created_at = _now_iso() rt.updated_at = _now_iso() session.add(rt) @@ -85,6 +91,7 @@ class ResourceInsert: """Benchmark inserting resource records.""" def setup(self) -> None: + self._batch_ctr = 0 self.engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(self.engine) self.session_factory = sessionmaker(bind=self.engine) @@ -104,16 +111,19 @@ class ResourceInsert: self.engine.dispose() def time_insert_100_resources(self) -> None: + offset = self._batch_ctr * 100 + self._batch_ctr += 1 session = self.session_factory() for i in range(100): + uid = offset + i r = ResourceModel() - r.resource_id = _make_ulid(i) - r.namespaced_name = f"bench/resource-{i}" + r.resource_id = _make_ulid(uid) + r.namespaced_name = f"bench/resource-{uid}" r.namespace = "bench" r.type_name = "bench/git-checkout" r.resource_kind = "physical" - r.location = f"/path/to/resource-{i}" - r.properties_json = json.dumps({"index": i}) + r.location = f"/path/to/resource-{uid}" + r.properties_json = json.dumps({"index": uid}) r.created_at = _now_iso() r.updated_at = _now_iso() session.add(r) diff --git a/benchmarks/uow_lifecycle_bench.py b/benchmarks/uow_lifecycle_bench.py index 9be54a7f2..e327ce2c7 100644 --- a/benchmarks/uow_lifecycle_bench.py +++ b/benchmarks/uow_lifecycle_bench.py @@ -188,11 +188,11 @@ class TimeUowActionPlanRoundTrip: session.close() def time_create_action_and_plan_via_uow(self) -> None: + # Reuse the pre-seeded action to avoid FK issues session = self.sf() ctx = UnitOfWorkContext(session) - uid = _bench_ulid() - action_name = f"local/bench-combo-{uid}" - ctx.actions.create(_make_bench_action(action_name)) - ctx.lifecycle_plans.create(_make_bench_plan(action_name=action_name)) + ctx.lifecycle_plans.create( + _make_bench_plan(action_name="local/bench-uow-action"), + ) session.commit() session.close() diff --git a/features/steps/project_repository_steps.py b/features/steps/project_repository_steps.py index 953f559b6..b0d80c707 100644 --- a/features/steps/project_repository_steps.py +++ b/features/steps/project_repository_steps.py @@ -75,13 +75,20 @@ def step_pr_init_db(context: Any) -> None: sf = sessionmaker(bind=engine) context.pr_engine = engine context.pr_session_factory = sf - context.pr_session = sf() + session = sf() + context.pr_session = session # Set up the resource type for FK constraints _setup_resource_type_pr(context.pr_session) - context.pr_project_repo = NamespacedProjectRepository(session_factory=sf) - context.pr_link_repo = ProjectResourceLinkRepository(session_factory=sf) + # Use a lambda that always returns the SAME session so that + # context.pr_session.commit() commits data flushed by the repos. + context.pr_project_repo = NamespacedProjectRepository( + session_factory=lambda: session, + ) + context.pr_link_repo = ProjectResourceLinkRepository( + session_factory=lambda: session, + ) context.pr_error = None context.pr_result = None context.pr_project = None