From 7df52b43dcd322908e78fb6b56c53ff043fabbcf Mon Sep 17 00:00:00 2001 From: "Brent E. Edwards" Date: Mon, 23 Mar 2026 09:00:44 +0000 Subject: [PATCH] =?UTF-8?q?test:=20add=20TDD=20bug-capture=20test=20for=20?= =?UTF-8?q?#1076=20=E2=80=94=20use=5Faction=20automation=5Fprofile=20propa?= =?UTF-8?q?gation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Behave BDD scenarios that capture the bug described in #1076 where PlanLifecycleService.use_action() does not resolve or propagate the automation_profile from the Action (or any other source in the spec precedence chain) to the created Plan. Three scenarios prove the bug exists: 1. Action with automation_profile="full-auto" — Plan.automation_profile is None instead of AutomationProfileRef(profile_name="full-auto", provenance=ACTION). 2. Action without automation_profile but with project-scoped config "trusted" — Plan.automation_profile is None instead of AutomationProfileRef(profile_name="trusted", provenance=PROJECT). 3. Action without automation_profile — Plan.automation_profile is None instead of resolving to the global default "supervised" with provenance=GLOBAL. All scenarios are tagged @tdd_expected_fail @tdd_bug @tdd_bug_1076. The @tdd_expected_fail tag inverts the result so CI passes while the bug is unfixed. Once the fix in #1076 is merged, the @tdd_expected_fail tag will be removed and the tests will run normally as regression guards. Robot integration tests are N/A — the bug is purely in the service layer with no CLI, I/O, or external dependencies involved. Duplicate When step bodies extracted into a shared _use_action_on_project helper. Guard assertion added after create_action to verify the Action itself stored the automation_profile. CHANGELOG.md entry added per CONTRIBUTING.md requirement. ISSUES CLOSED: #1098 --- CHANGELOG.md | 8 + ...tdd_use_action_automation_profile_steps.py | 216 ++++++++++++++++++ .../tdd_use_action_automation_profile.feature | 52 +++++ 3 files changed, 276 insertions(+) create mode 100644 features/steps/tdd_use_action_automation_profile_steps.py create mode 100644 features/tdd_use_action_automation_profile.feature diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0f04bfe..26cc78a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- Added TDD bug-capture tests for bug #1076 — `use_action()` does not + propagate `automation_profile` to Plan. Three Behave BDD scenarios + (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence + chain (action, project-scoped config, global default) for automation + profile resolution at `plan use` time. Tests prove the bug exists: the + Plan's `automation_profile` is always `None` regardless of the Action's + profile, project config, or global default. The `@tdd_expected_fail` tag + inverts this to a CI pass until the fix is merged. (#1098) - Added BuiltinAdapter class and MCP automatic resource slot creation. BuiltinAdapter wraps register_file_tools/register_git_tools/register_subplan_tool into a unified adapter interface. McpAdapter.infer_resource_slots() analyzes diff --git a/features/steps/tdd_use_action_automation_profile_steps.py b/features/steps/tdd_use_action_automation_profile_steps.py new file mode 100644 index 000000000..dbab461fa --- /dev/null +++ b/features/steps/tdd_use_action_automation_profile_steps.py @@ -0,0 +1,216 @@ +"""Step definitions for TDD Bug #1076 — use_action automation_profile propagation. + +These steps exercise ``PlanLifecycleService.use_action()`` and verify that +it resolves the automation profile using the spec's precedence chain +(plan > action > project > global) and sets the resolved profile as an +``AutomationProfileRef`` on the created Plan. + +On ``master`` (before the fix), ``use_action()`` constructs the ``Plan()`` +without passing ``automation_profile`` to the constructor. The Plan's +``automation_profile`` field is always ``None`` regardless of the Action's +``automation_profile`` value or any other configuration source. + +The assertions in these steps will **fail** until the bug is fixed, +proving the bug exists. The ``@tdd_expected_fail`` tag inverts the +result so CI passes. + +Bug #1076 — captures the test for automation_profile propagation from +the Action to the Plan via use_action(). Uses @tdd_expected_fail until +the fix in #1076 is merged. +""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.application.services.config_service import ConfigService +from cleveragents.application.services.plan_lifecycle_service import ( + PlanLifecycleService, +) +from cleveragents.config.settings import Settings +from cleveragents.domain.models.core.plan import ( + AutomationProfileProvenance, + Plan, + ProjectLink, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _use_action_on_project(context: Context, project: str) -> None: + """Use the current action on *project* and store the resulting plan. + + Shared by both profiled- and unprofiled-action When steps so that + the ``use_action`` call-site is defined in exactly one place. + """ + links: list[ProjectLink] = [ProjectLink(project_name=project)] + plan: Plan = context.ap_lifecycle_service.use_action( + action_name=str(context.ap_action.namespaced_name), + project_links=links, + ) + context.ap_plan = plan + + +# --------------------------------------------------------------------------- +# Given steps +# --------------------------------------------------------------------------- + + +@given("a plan lifecycle service for automation profile testing") +def step_create_lifecycle_service(context: Context) -> None: + """Create a PlanLifecycleService instance for automation profile tests.""" + settings: Settings = Settings() + context.ap_lifecycle_service = PlanLifecycleService(settings=settings) + + +@given('an available action "{name}" with automation_profile "{profile}"') +def step_create_action_with_profile(context: Context, name: str, profile: str) -> None: + """Create an available action with a specified automation_profile.""" + context.ap_action = context.ap_lifecycle_service.create_action( + name=name, + description=f"Test action {name} with automation profile", + definition_of_done="Test definition of done", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + automation_profile=profile, + ) + # Guard: verify the Action itself stored the profile (not silently dropped) + assert context.ap_action.automation_profile == profile, ( + f"Action.automation_profile is {context.ap_action.automation_profile!r}, " + f"expected {profile!r}. create_action() did not persist the profile " + f"on the Action model." + ) + + +@given('an available action "{name}" without automation_profile') +def step_create_action_without_profile(context: Context, name: str) -> None: + """Create an available action without an automation_profile.""" + context.ap_action = context.ap_lifecycle_service.create_action( + name=name, + description=f"Test action {name} without automation profile", + definition_of_done="Test definition of done", + strategy_actor="openai/gpt-4", + execution_actor="openai/gpt-4", + ) + + +@given('a project-scoped automation_profile "{profile}" for project "{project}"') +def step_set_project_scoped_profile( + context: Context, profile: str, project: str +) -> None: + """Configure a project-scoped automation_profile via ConfigService. + + This sets ``core.automation-profile`` at the project level so that + the precedence chain (plan > action > **project** > global) has a + value at level 3. ``use_action()`` should consult this when the + action itself has no automation_profile. + """ + tmpdir: str = tempfile.mkdtemp() + context.ap_config_tmpdir = tmpdir + tmp_path: Path = Path(tmpdir) + config_svc: ConfigService = ConfigService( + config_dir=tmp_path, + config_path=tmp_path / "config.toml", + ) + config_svc.set_project_value(project, "core.automation-profile", profile) + context.ap_config_service = config_svc + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when('I use the profiled action on project "{project}"') +def step_use_profiled_action(context: Context, project: str) -> None: + """Use the profiled action on a project via use_action().""" + _use_action_on_project(context, project) + + +@when('I use the unprofiled action on project "{project}"') +def step_use_unprofiled_action(context: Context, project: str) -> None: + """Use the unprofiled action on a project via use_action().""" + _use_action_on_project(context, project) + + +@when('I use the unprofiled action on configured project "{project}"') +def step_use_unprofiled_action_on_configured_project( + context: Context, project: str +) -> None: + """Use the unprofiled action on a project that has project-scoped config.""" + _use_action_on_project(context, project) + + +# --------------------------------------------------------------------------- +# Then steps +# --------------------------------------------------------------------------- + + +@then("the created plan automation_profile should not be None") +def step_plan_profile_not_none(context: Context) -> None: + """Assert the plan's automation_profile is set. + + Bug #1076: ``use_action()`` does not pass ``automation_profile`` to + the ``Plan()`` constructor. The resulting ``plan.automation_profile`` + is always ``None`` regardless of the Action's ``automation_profile`` + value or any other configuration source in the precedence chain + (plan > action > project > global). + """ + plan: Plan = context.ap_plan + assert plan.automation_profile is not None, ( + "Plan automation_profile is None. " + "PlanLifecycleService.use_action() does not resolve or " + "propagate the automation profile from the precedence chain " + "(plan > action > project > global) to the created Plan " + "(bug #1076). Expected an AutomationProfileRef, got None." + ) + + +@then('the created plan automation_profile name should be "{expected}"') +def step_plan_profile_name(context: Context, expected: str) -> None: + """Assert the plan's automation_profile has the expected profile name. + + Bug #1076: Since automation_profile is always None, the profile name + is never set. + """ + plan: Plan = context.ap_plan + assert plan.automation_profile is not None, ( + "Plan automation_profile is None — cannot verify profile name. " + "use_action() does not propagate automation_profile (bug #1076)." + ) + actual: str = plan.automation_profile.profile_name + assert actual == expected, ( + f"Plan automation_profile.profile_name is '{actual}', " + f"expected '{expected}'. The precedence resolution in " + f"use_action() did not resolve the correct profile (bug #1076)." + ) + + +@then('the created plan automation_profile provenance should be "{expected}"') +def step_plan_profile_provenance(context: Context, expected: str) -> None: + """Assert the plan's automation_profile has the expected provenance. + + Bug #1076: Since automation_profile is always None, the provenance + source is never set. + """ + plan: Plan = context.ap_plan + assert plan.automation_profile is not None, ( + "Plan automation_profile is None — cannot verify provenance. " + "use_action() does not propagate automation_profile (bug #1076)." + ) + expected_provenance: AutomationProfileProvenance = AutomationProfileProvenance( + expected + ) + actual: AutomationProfileProvenance = plan.automation_profile.provenance + assert actual == expected_provenance, ( + f"Plan automation_profile.provenance is '{actual.value}', " + f"expected '{expected_provenance.value}'. The precedence " + f"resolution in use_action() did not record the correct " + f"provenance source (bug #1076)." + ) diff --git a/features/tdd_use_action_automation_profile.feature b/features/tdd_use_action_automation_profile.feature new file mode 100644 index 000000000..7b890b10e --- /dev/null +++ b/features/tdd_use_action_automation_profile.feature @@ -0,0 +1,52 @@ +@tdd_expected_fail @tdd_bug @tdd_bug_1076 +Feature: TDD Bug #1076 — use_action() does not propagate automation_profile to Plan + As a developer + I want to verify that use_action() resolves the automation profile + from the precedence chain and sets it on the created Plan + So that the bug is captured and will be caught by a regression test + + Per the specification (docs/specification.md): + + - Line 18919: "The resolved automation profile name for this plan [...] + Determined at `plan use` time using the profile precedence rules + (plan > action > project > global). Once set, it is locked to the plan." + + - Line 18967: "2. The plan's automation profile is resolved + (plan > action > project > global precedence)" + + Currently, PlanLifecycleService.use_action() constructs the Plan without + passing automation_profile to the Plan() constructor. The Plan's + automation_profile field is always None regardless of the action's + automation_profile value or any other configuration source. + + These tests assert the expected behavior and will FAIL until the bug is + fixed. The @tdd_expected_fail tag inverts the result so CI passes. + + # Bug #1076 — captures the test for automation_profile propagation from + # the Action to the Plan via use_action(). Uses @tdd_expected_fail until + # the fix in #1076 is merged. + + Scenario: Plan inherits automation_profile from action when action has a profile set + Given a plan lifecycle service for automation profile testing + And an available action "local/profiled-action" with automation_profile "full-auto" + When I use the profiled action on project "test-project" + Then the created plan automation_profile should not be None + And the created plan automation_profile name should be "full-auto" + And the created plan automation_profile provenance should be "action" + + Scenario: Plan gets project-scoped automation_profile when action has no profile + Given a plan lifecycle service for automation profile testing + And an available action "local/project-config-action" without automation_profile + And a project-scoped automation_profile "trusted" for project "configured-project" + When I use the unprofiled action on configured project "configured-project" + Then the created plan automation_profile should not be None + And the created plan automation_profile name should be "trusted" + And the created plan automation_profile provenance should be "project" + + Scenario: Plan gets global default automation_profile when action has no profile + Given a plan lifecycle service for automation profile testing + And an available action "local/unprofiled-action" without automation_profile + When I use the unprofiled action on project "test-project" + Then the created plan automation_profile should not be None + And the created plan automation_profile name should be "supervised" + And the created plan automation_profile provenance should be "global" -- 2.52.0