forked from HAL9000/cleveragents-core
02250473ad
Fix all failing CI quality gates (lint, unit_tests, format) without suppressing any quality enforcement. Root causes and fixes: 1. Format: features/steps/plan_namespaced_name_tdd_steps.py had trailing whitespace; fixed by running ruff format. 2. Unit tests - A2A JSON-RPC 2.0 migration (commit9c6d6915) renamed A2aRequest fields (operation→method, request_id→id, a2a_version→jsonrpc) and A2aResponse fields (status+data→result, request_id→id) but did not update all step files and feature files: - a2a_jsonrpc_wire_format_steps.py: added use_step_matcher('re') and reset to 'parse' at end to prevent parallel test interference - a2a_facade_wiring_steps.py: updated operation= to method=, .status/.data to .result - a2a_facade_steps.py: updated request_id→id, a2a_version→jsonrpc, A2aResponse(request_id=..., status=...) to new API - m6_facade_steps.py: updated all old API usage - devcontainer_cleanup_steps.py: updated A2aRequest(operation=...) - plan_prompt_command_steps.py: updated A2aRequest(operation=...) - wf03_plan_prompt_confidence_steps.py: updated A2aRequest(operation=...) - consolidated_misc.feature: updated old A2aRequest/A2aResponse scenarios 3. Unit tests - Session CLI output changed (commit0d5d9cf0and others): - 'Session Created' → 'Session created' (lowercase) - 'Session Details' → 'Session Summary' - 'Sessions (N total)' → 'Sessions' - session list JSON: top-level 'total' → nested 'summary.total' - Fixed in: session_cli.feature, session_cli_coverage_boost.feature, session_cli_uncovered_branches.feature, session_list_error.feature, tdd_session_create_persist_steps.py 4. Unit tests - Plan list output changed (commit1a07a891): - 'V3 Lifecycle Plans' → 'Plans' - 'Lifecycle Plans' → 'Plans' - Name column removed (restored in source) - Invariants column removed (restored in source) - Project truncation removed (restored in source) - Fixed in: plan_cli_cancel_revert_coverage.feature, plan_lifecycle_cli_coverage.feature, plan_cli_coverage_boost_steps.py, plan.py (source code restored) 5. Unit tests - Plan apply command now requires ULID (commit300a5d6d): - plan_cli_coverage_r3.feature: updated 'PLAN-001' to valid ULID - plan_cli_coverage_r3_steps.py: added --yes flag, added new step for no-eligible-plans path 6. Unit tests - Various source code bugs: - ThoughtBlock: converted from @dataclass to Pydantic BaseModel (architecture test requires all dataclasses to use Pydantic) - session.py: added DatabaseError handling to export, import, tell commands - database.py: fixed rollback_to() to reuse checkpoint connection for writes - database.py: added _get_checkpoint_conn() helper - check-tls-cert.py: fixed SSLCertVerificationError.reason AttributeError 7. Unit tests - Test step bugs: - error_recovery_coverage_boost_steps.py: fixed invalid ULID _PLAN_ID - session_service_coverage_steps.py: fixed 'sha256:' prefix bug in checksum - database_models_new_coverage_steps.py: added 'name' field to session mock - async_audit_recording_steps.py: fixed Settings(audit_async=False) via env var - coverage_threshold_config_steps.py: added --coverage-min pattern support - m5_acms_smoke_steps.py: updated usage hint text - actor_cli_yaml_steps.py: updated 'Removed actor' → 'Actor removed' - aimodelscredentials_steps.py: set context.imported_class in import step - domain_base_model.feature: added missing 'When I examine model_config' step - tui_first_run_steps.py: fixed module reload to restore cleveragents.tui.* modules after test (prevented patch interference in subsequent tests) - tui_first_run_steps.py: added set_search('') step for empty string - resource_handler_base_coverage_r3_steps.py: use _MinimalHandler instead of DatabaseResourceHandler for NotImplementedError tests - resource_handler_crud.feature: updated to test new DatabaseHandler behavior - resource_handler_sandbox.feature: updated to test new DatabaseHandler behavior - tdd_json_decode_crash_persistence.feature: fixed @tdd_bug → @tdd_issue tags 8. Parallel test interference: - All step files using use_step_matcher('re') now reset to 'parse' at end to prevent global matcher state leaking to subsequent step files
261 lines
10 KiB
Python
261 lines
10 KiB
Python
"""Step definitions for WF03 plan prompt and confidence-threshold pausing.
|
|
|
|
Covers:
|
|
- AC1 (H1): plan prompt via A2A facade (S15822)
|
|
- AC2 (H2): confidence-threshold pausing with cautious profile (S37262-37367)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
from cleveragents.a2a.facade import A2aLocalFacade
|
|
from cleveragents.a2a.models import A2aRequest
|
|
from cleveragents.domain.models.core.automation_profile import BUILTIN_PROFILES
|
|
from cleveragents.domain.models.core.escalation import (
|
|
ConfidenceFactors,
|
|
OperationContext,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC1: plan prompt via A2A facade
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("a plan prompt facade with no services")
|
|
def step_plan_prompt_facade(context: Context) -> None:
|
|
"""Create a facade with no wired services for plan prompt tests."""
|
|
context.wf03_facade = A2aLocalFacade()
|
|
|
|
|
|
@when('I dispatch plan prompt for plan "{plan_id}" with guidance "{guidance}"')
|
|
def step_dispatch_plan_prompt(context: Context, plan_id: str, guidance: str) -> None:
|
|
"""Dispatch a plan prompt operation via the facade."""
|
|
request = A2aRequest(
|
|
method="_cleveragents/plan/prompt",
|
|
params={"plan_id": plan_id, "guidance": guidance},
|
|
)
|
|
context.wf03_response = context.wf03_facade.dispatch(request)
|
|
|
|
|
|
@when('I dispatch plan prompt for plan "{plan_id}" with empty guidance')
|
|
def step_dispatch_plan_prompt_empty(context: Context, plan_id: str) -> None:
|
|
"""Dispatch a plan prompt with empty guidance string."""
|
|
request = A2aRequest(
|
|
method="_cleveragents/plan/prompt",
|
|
params={"plan_id": plan_id, "guidance": ""},
|
|
)
|
|
context.wf03_response = context.wf03_facade.dispatch(request)
|
|
|
|
|
|
@then('the plan prompt response status should be "{status}"')
|
|
def step_plan_prompt_status(context: Context, status: str) -> None:
|
|
"""Assert the plan prompt response data status."""
|
|
resp = context.wf03_response
|
|
actual = (resp.result or {}).get("status", "")
|
|
assert actual == status, (
|
|
f"Expected data.status '{status}', got '{actual}' in {(resp.result or {})}"
|
|
)
|
|
|
|
|
|
@then('the plan prompt response should contain the plan id "{plan_id}"')
|
|
def step_plan_prompt_contains_plan_id(context: Context, plan_id: str) -> None:
|
|
"""Assert plan_id appears in the response data."""
|
|
resp = context.wf03_response
|
|
actual = (resp.result or {}).get("plan_id", "")
|
|
assert actual == plan_id, (
|
|
f"Expected plan_id '{plan_id}', got '{actual}' in {(resp.result or {})}"
|
|
)
|
|
|
|
|
|
@then('the plan prompt response should echo the guidance "{guidance}"')
|
|
def step_plan_prompt_echoes_guidance(context: Context, guidance: str) -> None:
|
|
"""Assert guidance text propagates through the dispatch path."""
|
|
resp = context.wf03_response
|
|
actual = (resp.result or {}).get("guidance", "")
|
|
assert actual == guidance, (
|
|
f"Expected guidance '{guidance}', got '{actual}' in {(resp.result or {})}"
|
|
)
|
|
|
|
|
|
@then("the plan prompt response should echo empty guidance")
|
|
def step_plan_prompt_echoes_empty_guidance(context: Context) -> None:
|
|
"""Assert guidance is empty string in the response."""
|
|
resp = context.wf03_response
|
|
actual = (resp.result or {}).get("guidance", "")
|
|
assert actual == "", (
|
|
f"Expected empty guidance, got '{actual}' in {(resp.result or {})}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC2: confidence-threshold pausing with cautious profile
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@given("the cautious edit_code threshold should be {threshold:g}")
|
|
def step_verify_cautious_threshold(context: Context, threshold: float) -> None:
|
|
"""Verify the cautious profile edit_code threshold."""
|
|
profile = BUILTIN_PROFILES["cautious"]
|
|
actual = profile.edit_code
|
|
assert abs(actual - threshold) < 1e-6, (
|
|
f"Expected cautious edit_code={threshold}, got {actual}"
|
|
)
|
|
|
|
|
|
@when(
|
|
'I evaluate escalation for operation "{op}" '
|
|
"with factors producing confidence {target:g}"
|
|
)
|
|
def step_evaluate_with_target_confidence(
|
|
context: Context, op: str, target: float
|
|
) -> None:
|
|
"""Evaluate escalation with factors tuned to produce a specific confidence.
|
|
|
|
The AutonomyController formula:
|
|
score = w_psr*psr + w_cf*cf + w_risk*(1-risk) + w_ic*(1-ic)
|
|
with default weights [0.30, 0.20, 0.30, 0.20].
|
|
|
|
To produce target T: set psr=T, cf=T, risk=(1-T), ic=(1-T).
|
|
Then score = 0.3*T + 0.2*T + 0.3*(1-(1-T)) + 0.2*(1-(1-T))
|
|
= 0.3*T + 0.2*T + 0.3*T + 0.2*T = T.
|
|
"""
|
|
factors = ConfidenceFactors(
|
|
past_success_rate=target,
|
|
codebase_familiarity=target,
|
|
risk_assessment=1.0 - target,
|
|
invariant_complexity=1.0 - target,
|
|
)
|
|
operation = OperationContext(operation_type=op)
|
|
context.decision = context.controller.should_proceed_automatically(
|
|
operation=operation,
|
|
factors=factors,
|
|
profile=context.profile,
|
|
)
|
|
context.confidence = context.decision.confidence
|
|
|
|
|
|
@then("the escalation confidence should be below the cautious threshold {threshold:g}")
|
|
def step_confidence_below_threshold(context: Context, threshold: float) -> None:
|
|
"""Assert confidence is strictly below the given threshold."""
|
|
assert context.decision is not None
|
|
assert context.decision.confidence < threshold, (
|
|
f"Expected confidence < {threshold}, got {context.decision.confidence:.3f}"
|
|
)
|
|
|
|
|
|
@then(
|
|
"the escalation confidence should be at or above the cautious "
|
|
"threshold {threshold:g}"
|
|
)
|
|
def step_confidence_at_or_above_threshold(context: Context, threshold: float) -> None:
|
|
"""Assert confidence is at or above the given threshold."""
|
|
assert context.decision is not None
|
|
assert context.decision.confidence >= threshold, (
|
|
f"Expected confidence >= {threshold}, got {context.decision.confidence:.3f}"
|
|
)
|
|
|
|
|
|
@then("the decision explanation should mention threshold")
|
|
def step_explanation_mentions_threshold(context: Context) -> None:
|
|
"""Assert the explanation references the threshold."""
|
|
assert context.decision is not None
|
|
explanation = context.decision.explanation.lower()
|
|
assert "threshold" in explanation, (
|
|
f"Expected 'threshold' in explanation: {context.decision.explanation}"
|
|
)
|
|
|
|
|
|
@then("the decision explanation should mention escalation to user")
|
|
def step_explanation_mentions_escalation(context: Context) -> None:
|
|
"""Assert the explanation references escalation / user involvement."""
|
|
assert context.decision is not None
|
|
explanation = context.decision.explanation.lower()
|
|
assert any(
|
|
word in explanation for word in ("escalat", "user", "manual", "human")
|
|
), f"Expected escalation mention in: {context.decision.explanation}"
|
|
|
|
|
|
@given('a decision seeded with confidence {conf:g} for operation "{op}"')
|
|
def step_seed_decision_for_pause_check(context: Context, conf: float, op: str) -> None:
|
|
"""Prepare factors to produce a given confidence for an operation."""
|
|
context.seeded_confidence = conf
|
|
context.seeded_operation = op
|
|
context.seeded_factors = ConfidenceFactors(
|
|
past_success_rate=conf,
|
|
codebase_familiarity=conf,
|
|
risk_assessment=1.0 - conf,
|
|
invariant_complexity=1.0 - conf,
|
|
)
|
|
|
|
|
|
@when("I check whether the system should proceed automatically")
|
|
def step_check_proceed(context: Context) -> None:
|
|
"""Run the autonomy controller check with the seeded values."""
|
|
operation = OperationContext(operation_type=context.seeded_operation)
|
|
context.decision = context.controller.should_proceed_automatically(
|
|
operation=operation,
|
|
factors=context.seeded_factors,
|
|
profile=context.profile,
|
|
)
|
|
context.confidence = context.decision.confidence
|
|
# Store plan_id context for the pause-and-resume scenario
|
|
context.paused_plan_id = f"PLAN-WF03-{context.seeded_operation}"
|
|
|
|
|
|
@when('I provide plan prompt guidance "{guidance}" for the paused plan')
|
|
def step_provide_guidance_for_paused_plan(context: Context, guidance: str) -> None:
|
|
"""Provide guidance via plan prompt using plan_id from paused decision."""
|
|
facade = A2aLocalFacade()
|
|
request = A2aRequest(
|
|
method="_cleveragents/plan/prompt",
|
|
params={
|
|
"plan_id": context.paused_plan_id,
|
|
"guidance": guidance,
|
|
},
|
|
)
|
|
context.wf03_guidance_response = facade.dispatch(request)
|
|
|
|
|
|
@then("the guidance should be recorded as a user intervention")
|
|
def step_guidance_is_user_intervention(context: Context) -> None:
|
|
"""Assert guidance was accepted by the facade."""
|
|
resp = context.wf03_guidance_response
|
|
assert resp.result is not None, f"Expected status 'ok', got error: {resp.error}"
|
|
status = (resp.result or {}).get("status", "")
|
|
assert status == "guidance_injected", (
|
|
f"Expected data.status 'guidance_injected', got '{status}' in {(resp.result or {})}"
|
|
)
|
|
|
|
|
|
@then("the guidance response plan id should match the paused decision context")
|
|
def step_guidance_plan_id_matches_paused(context: Context) -> None:
|
|
"""Assert plan prompt response plan_id matches paused context."""
|
|
resp = context.wf03_guidance_response
|
|
actual = (resp.result or {}).get("plan_id", "")
|
|
expected = context.paused_plan_id
|
|
assert actual == expected, (
|
|
f"Expected plan_id '{expected}' from paused context, got '{actual}'"
|
|
)
|
|
|
|
|
|
@when('I re-evaluate with corrected confidence {conf:g} for operation "{op}"')
|
|
def step_reevaluate_with_corrected_confidence(
|
|
context: Context, conf: float, op: str
|
|
) -> None:
|
|
"""Re-evaluate after guidance with higher confidence factors."""
|
|
factors = ConfidenceFactors(
|
|
past_success_rate=conf,
|
|
codebase_familiarity=conf,
|
|
risk_assessment=1.0 - conf,
|
|
invariant_complexity=1.0 - conf,
|
|
)
|
|
operation = OperationContext(operation_type=op)
|
|
context.decision = context.controller.should_proceed_automatically(
|
|
operation=operation,
|
|
factors=factors,
|
|
profile=context.profile,
|
|
)
|
|
context.confidence = context.decision.confidence
|