*** Settings ***
Documentation     E2E test for Workflow Example 10: Full-Auto Batch Operations.
...
...               A team reformats packages in a monorepo using the ``full-auto``
...               automation profile.  Multiple plans run without human
...               intervention (strategize → execute → apply automatically).
...               Includes a deliberately broken action (non-existent LLM actor)
...               to demonstrate batch error handling.
...
...               Requires real LLM API keys — zero mocking.
Library           Collections
Resource          common_e2e.resource
Suite Setup       WF10 Suite Setup
Suite Teardown    E2E Suite Teardown

*** Variables ***
@{PACKAGE_NAMES}      pkg_auth    pkg_common    pkg_billing
${ACTION_NAME}        local/format-codebase
${PLAN_TIMEOUT}       180s

*** Keywords ***
WF10 Suite Setup
    [Documentation]    E2E Suite Setup plus database initialisation.
    E2E Suite Setup
    # Initialise the database so CLI commands work in all tests.
    ${init}=    Run CleverAgents Command    init    --force    --yes
    Should Be Equal As Integers    ${init.rc}    0

Create Package Directory
    [Documentation]    Create a single package with badly-formatted Python files.
    [Arguments]    ${monorepo}    ${pkg_name}
    ${pkg_dir}=    Set Variable    ${monorepo}${/}${pkg_name}
    Create Directory    ${pkg_dir}${/}src
    # __init__.py with extra blank lines and bad spacing
    ${init_content}=    Set Variable
    ...    \n\n\n"""${pkg_name} package."""\n\n\nimport os\nimport sys\nimport json\n\n\n__all__=["main"]\n
    Create File    ${pkg_dir}${/}src${/}__init__.py    ${init_content}
    # main.py with intentionally bad formatting: unsorted imports, extra spaces, long lines
    ${main_content}=    Set Variable
    ...    import json\nimport os\nimport sys\nfrom pathlib import Path\nimport re\n\n\ndef   main( ):\n    """Entry point with   bad   formatting."""\n    x=1\n    y =   2\n    z=x+y\n    data = {"key":   "value",  "another":   "item"}\n    return z\n\nif __name__=="__main__":\n    main( )\n
    Create File    ${pkg_dir}${/}src${/}main.py    ${main_content}
    RETURN    ${pkg_dir}

Create Temp Monorepo
    [Documentation]    Create a temporary monorepo with multiple badly-formatted packages.
    ...
    ...                Each healthy package contains ``src/__init__.py`` and ``src/main.py``
    ...                with intentionally poor formatting (extra spaces, unsorted imports).
    ...                Returns the path to the monorepo root and the detected branch name.
    ${monorepo}=    Create Temp Git Repo    wf10-monorepo
    FOR    ${pkg_name}    IN    @{PACKAGE_NAMES}
        Create Package Directory    ${monorepo}    ${pkg_name}
    END
    # Commit all packages so git-checkout resources have content
    ${add_res}=    Run Process    git    add    .    cwd=${monorepo}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${add_res.rc}    0    git add failed: ${add_res.stderr}
    ${commit_res}=    Run Process    git    commit    -m    Add packages with bad formatting    cwd=${monorepo}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${commit_res.rc}    0    git commit failed: ${commit_res.stderr}
    # Detect the actual default branch name (may be main or master)
    ${branch_result}=    Run Process    git    rev-parse    --abbrev-ref    HEAD    cwd=${monorepo}    timeout=60s    on_timeout=kill
    Should Be Equal As Integers    ${branch_result.rc}    0    git rev-parse failed: ${branch_result.stderr}
    ${branch}=    Strip String    ${branch_result.stdout}
    Log    Detected monorepo branch: ${branch}
    RETURN    ${monorepo}    ${branch}

Write Action Config
    [Documentation]    Write a formatting action YAML config with full-auto profile.
    ...
    ...                Uses dynamic actor selection based on available API keys.
    ...                Returns the path to the YAML file.
    # Pick an actor that matches the available API key
    ${has_anthropic}=    Evaluate    bool(__import__('os').environ.get('ANTHROPIC_API_KEY', ''))
    IF    ${has_anthropic}
        ${actor}=    Set Variable    anthropic/claude-sonnet-4-20250514
    ELSE
        ${actor}=    Set Variable    openai/gpt-4o
    END
    ${yaml_path}=    Set Variable    ${SUITE_HOME}${/}format_action.yaml
    ${config}=    Catenate    SEPARATOR=\n
    ...    name: ${ACTION_NAME}
    ...    description: "Reformat Python source files for consistent style"
    ...    strategy_actor: ${actor}
    ...    execution_actor: ${actor}
    ...    definition_of_done: >
    ...        All Python files are consistently formatted without creating new files.
    ...        Creating .flake8, .pre-commit-config.yaml or a new pyproject.toml
    ...        file is forbidden.
    ...    automation_profile: full-auto
    ...    reusable: true
    ...    state: available
    ...    read_only: false
    ...    invariants:
    ...    ${SPACE}${SPACE}- "Changes must be whitespace-only (no semantic modifications)"
    ...    ${SPACE}${SPACE}- "Every package must pass its own test suite after formatting"
    Create File    ${yaml_path}    ${config}\n
    RETURN    ${yaml_path}

Clean Workspace Template Files
    [Documentation]    Remove template files from the workspace and monorepo
    ...                that the LLM may regenerate during formatting, to prevent
    ...                add/add merge conflicts during plan apply.
    ...
    ...                CleverAgents copies workspace template files into project
    ...                directories during ``project create``, so this keyword
    ...                must run **after** project registration and **before**
    ...                plan launch.  It also runs ``git clean -fd`` in the
    ...                monorepo to remove any untracked files that were copied.
    [Arguments]    ${target_dir}=${SUITE_HOME}
    @{conflicting}=    Create List
    ...    .flake8
    ...    .pre-commit-config.yaml
    ...    pyproject.toml
    ...    requirements-dev.txt
    ...    requirements.txt
    ...    setup.py
    ...    setup.cfg
    ...    tox.ini
    FOR    ${file}    IN    @{conflicting}
        ${path}=    Set Variable    ${target_dir}${/}${file}
        Run Keyword And Ignore Error    Remove File    ${path}
    END
    # Also run git clean to remove any untracked files/directories that
    # the workspace template may have deposited in the monorepo.
    ${git_clean}=    Run Process    git    clean    -fd    cwd=${target_dir}    timeout=60s    on_timeout=kill
    Log    git clean in ${target_dir}: rc=${git_clean.rc}    level=DEBUG

Write Broken Action Config
    [Documentation]    Write a deliberately broken action YAML that uses a non-existent
    ...                LLM actor.  Plans created with this action will fail during
    ...                execution when the strategy/execution actor cannot be resolved.
    ${yaml_path}=    Set Variable    ${SUITE_HOME}${/}broken_action.yaml
    ${config}=    Catenate    SEPARATOR=\n
    ...    name: local/broken-format
    ...    description: "Deliberately broken action for error handling testing"
    ...    strategy_actor: nonexistent/model-xyz-404
    ...    execution_actor: nonexistent/model-xyz-404
    ...    definition_of_done: "This action should always fail"
    ...    automation_profile: full-auto
    ...    reusable: true
    ...    state: available
    ...    read_only: false
    ...    invariants:
    ...    ${SPACE}${SPACE}- "No changes expected — action is broken"
    Create File    ${yaml_path}    ${config}\n
    RETURN    ${yaml_path}

Register Package Resources And Projects
    [Documentation]    Register git-checkout resources and create projects for the
    ...                specified packages.
    ...
    ...                Error handling is tested separately via a broken action
    ...                (non-existent LLM actor), not via missing resources.
    ...
    ...                Resource/project names use fixed ``local/<pkg>`` prefixes without
    ...                UUID suffixes — safe because ``init --force --yes`` in suite setup
    ...                resets the workspace database before each run.
    [Arguments]    ${monorepo}    ${branch}    @{healthy_packages}
    FOR    ${pkg_name}    IN    @{healthy_packages}
        # Register git-checkout resource pointing to the monorepo root
        ${res_result}=    Run CleverAgents Command
        ...    resource    add    git-checkout    local/${pkg_name}
        ...    --path    ${monorepo}    --branch    ${branch}
        Log    Resource ${pkg_name}: ${res_result.stdout}
        Should Not Contain    ${res_result.stderr}    Traceback
        ...    Resource registration for ${pkg_name} produced Traceback:\n${res_result.stderr}
        # Create project linked to resource — must succeed WITHOUT Traceback.
        ${proj_result}=    Run CleverAgents Command
        ...    project    create    local/${pkg_name}
        ...    --resource    local/${pkg_name}
        Log    Project ${pkg_name}: ${proj_result.stdout}
        Should Not Contain    ${proj_result.stderr}    Traceback
        ...    Project creation for ${pkg_name} produced Traceback:\n${proj_result.stderr}
    END

Launch Batch Plans
    [Documentation]    Launch a plan for each package in full-auto mode.
    ...
    ...                Returns a list of plan identifiers extracted from output.
    ...                Uses ``expected_rc=None`` because broken packages may fail
    ...                during plan use (which is expected behaviour).
    [Arguments]    @{all_packages}
    @{plan_ids}=    Create List
    FOR    ${pkg_name}    IN    @{all_packages}
        ${result}=    Run CleverAgents Command
        ...    plan    use    ${ACTION_NAME}    local/${pkg_name}
        ...    --automation-profile    full-auto    --format    plain
        ...    timeout=${PLAN_TIMEOUT}    expected_rc=None
        Log    Plan use ${pkg_name} stdout: ${result.stdout}
        Log    Plan use ${pkg_name} stderr: ${result.stderr}
        Should Not Contain    ${result.stderr}    Traceback
        ...    plan use for ${pkg_name} produced Traceback:\n${result.stderr}
        # Extract plan ID from output — must find a valid ULID (Crockford Base32)
        ${combined}=    Set Variable    ${result.stdout} ${result.stderr}
        ${match}=    Get Regexp Matches    ${combined}    ([0-9A-HJKMNP-TV-Z]{26})    flags=IGNORECASE
        ${match_count}=    Get Length    ${match}
        IF    ${match_count} > 0
            ${plan_id}=    Set Variable    ${match}[0]
            Append To List    ${plan_ids}    ${plan_id}
            Log    Captured plan ID for ${pkg_name}: ${plan_id}
        ELSE
            Log    No plan ID extracted for ${pkg_name} (rc=${result.rc}) — plan creation may have failed    WARN
        END
    END
    RETURN    @{plan_ids}

Execute Batch Plans
    [Documentation]    Execute each plan through the strategize→execute pipeline.
    ...
    ...                ``plan execute`` runs the current plan phase synchronously.
    ...                With the full-auto automation profile, a single ``plan execute``
    ...                call auto-advances through both strategize and execute phases,
    ...                leaving the plan ready for ``plan apply --yes``.
    ...
    ...                Plans that fail during execution (e.g. broken action with
    ...                non-existent actor) are logged but do not abort the batch —
    ...                the batch continues with remaining plans.
    ...
    ...                Returns a list of plan IDs that completed execution
    ...                successfully (candidates for apply).
    [Arguments]    @{plan_ids}
    @{executed_ids}=    Create List
    FOR    ${plan_id}    IN    @{plan_ids}
        Log    Executing plan ${plan_id} (strategize + execute via full-auto)
        ${exec}=    Run CleverAgents Command
        ...    plan    execute    ${plan_id}    --format    plain
        ...    timeout=${PLAN_TIMEOUT}    expected_rc=None
        Log    Execute ${plan_id} rc=${exec.rc}: ${exec.stdout}
        IF    ${exec.rc} != 0
            Log    Plan ${plan_id} failed during execution (rc=${exec.rc}): ${exec.stderr}    WARN
            CONTINUE
        END
        Append To List    ${executed_ids}    ${plan_id}
    END
    RETURN    @{executed_ids}

Apply Batch Plans
    [Documentation]    Apply each successfully-executed plan via ``plan apply --yes``.
    ...
    ...                After ``plan execute`` with full-auto profile, plans are in
    ...                the ``execute/complete`` state.  ``plan apply --yes`` performs
    ...                the actual application (e.g. committing changes to the repo)
    ...                and completes the apply phase, transitioning the plan to the
    ...                ``applied`` processing state.
    ...
    ...                Note: ``plan lifecycle-apply`` only transitions the plan INTO
    ...                the apply phase (``apply/queued``) without completing it.
    ...                ``plan apply --yes`` is required to actually run and complete
    ...                the apply step.
    ...
    ...                Plans that fail during apply are logged but do not abort
    ...                the batch.  Returns a list of plan IDs that were applied.
    [Arguments]    @{executed_ids}
    @{applied_ids}=    Create List
    FOR    ${plan_id}    IN    @{executed_ids}
        Log    Applying plan ${plan_id}
        ${apply}=    Run CleverAgents Command
        ...    plan    apply    --yes    ${plan_id}    --format    plain
        ...    timeout=${PLAN_TIMEOUT}    expected_rc=None
        Log    Apply ${plan_id} rc=${apply.rc}: ${apply.stdout}
        IF    ${apply.rc} != 0
            Log    Plan ${plan_id} failed during apply (rc=${apply.rc}) stdout: ${apply.stdout}    WARN
            Log    Plan ${plan_id} failed during apply (rc=${apply.rc}) stderr: ${apply.stderr}    WARN
            CONTINUE
        END
        Append To List    ${applied_ids}    ${plan_id}
    END
    RETURN    @{applied_ids}

*** Test Cases ***
Workflow 10 Full-Auto Batch Formatting
    [Documentation]    End-to-end test for full-auto batch formatting across
    ...                multiple packages in a monorepo, including error handling
    ...                when one plan fails due to a broken action.
    ...
    ...                1. Creates a monorepo with 3 badly-formatted Python packages
    ...                2. Registers a reusable formatting action with full-auto profile
    ...                3. Registers a broken action (non-existent LLM actor) for error testing
    ...                4. Registers resources and projects for healthy packages
    ...                5. Creates plans in full-auto mode via ``plan use``
    ...                6. Executes all plans via ``plan execute`` (strategize + execute)
    ...                7. Applies successful plans via ``plan apply --yes``
     ...                8. Verifies batch results via ``plan list`` with state filters
    ...                9. Verifies error handling — broken action's plan fails during execution
    [Tags]    E2E
    [Timeout]    25 minutes
    [Teardown]    Run CleverAgents Command    config    set    core.automation-profile    manual    expected_rc=None
    Skip If No LLM Keys

    # Prevent add/add merge conflicts during plan apply by removing workspace
    # template files that the LLM may regenerate.
    Clean Workspace Template Files

    # --- Step 1: Create temp monorepo with badly-formatted packages ---
    ${monorepo}    ${branch}=    Create Temp Monorepo
    Log    Monorepo created at: ${monorepo} (branch: ${branch})
    Directory Should Exist    ${monorepo}${/}pkg_auth${/}src
    Directory Should Exist    ${monorepo}${/}pkg_common${/}src
    Directory Should Exist    ${monorepo}${/}pkg_billing${/}src

    # --- Step 2: Create the formatting action with full-auto profile ---
    ${yaml_path}=    Write Action Config
    File Should Exist    ${yaml_path}
    ${action_result}=    Run CleverAgents Command
    ...    action    create    --config    ${yaml_path}
    Log    Action create output: ${action_result.stdout}
    Should Not Contain    ${action_result.stderr}    Traceback
    Output Should Contain    ${action_result}    format-codebase

    # Also create a broken action with a non-existent actor for error handling
    ${broken_action}=    Write Broken Action Config
    ${broken_action_result}=    Run CleverAgents Command
    ...    action    create    --config    ${broken_action}
    Log    Broken action create output: ${broken_action_result.stdout}
    Output Should Contain    ${broken_action_result}    broken-format

    # --- Step 3: Register resources and projects for all packages ---
    Register Package Resources And Projects    ${monorepo}    ${branch}    @{PACKAGE_NAMES}

    # Clean workspace template files that were copied into the monorepo
    # during project creation, to prevent add/add merge conflicts during apply.
    Clean Workspace Template Files    ${monorepo}

    # --- Step 4: Create plans — healthy + broken ---
    # 4a: Launch plans for healthy packages with the good action
    @{plan_ids}=    Launch Batch Plans    @{PACKAGE_NAMES}
    ${plan_count}=    Get Length    ${plan_ids}
    Log    Healthy plan IDs (${plan_count}): @{plan_ids}
    Should Be True    ${plan_count} == 3
    ...    Expected 3 healthy plan IDs but got ${plan_count}

    # 4b: Launch plan for first healthy package but with BROKEN action
    #     (non-existent actor will cause execution to fail)
    ${broken_result}=    Run CleverAgents Command
    ...    plan    use    local/broken-format    local/pkg_auth
    ...    --automation-profile    full-auto    --format    plain
    ...    timeout=${PLAN_TIMEOUT}    expected_rc=None
    Log    Broken action plan use rc=${broken_result.rc}: ${broken_result.stdout}
    Log    Broken action plan use stderr: ${broken_result.stderr}
    Should Not Contain    ${broken_result.stderr}    Traceback
    ${broken_plan_failed_at_creation}=    Evaluate    ${broken_result.rc} != 0
    ${broken_plan_id}=    Set Variable    ${EMPTY}
    IF    not ${broken_plan_failed_at_creation}
        # Extract plan ID for the broken plan
        ${combined}=    Set Variable    ${broken_result.stdout} ${broken_result.stderr}
        ${match}=    Get Regexp Matches    ${combined}    ([0-9A-HJKMNP-TV-Z]{26})    flags=IGNORECASE
        ${match_count}=    Get Length    ${match}
        IF    ${match_count} > 0
            ${broken_plan_id}=    Set Variable    ${match}[0]
            Append To List    ${plan_ids}    ${broken_plan_id}
            Log    Broken plan ID captured: ${broken_plan_id}
        END
    ELSE
        Log    Broken plan failed at creation (rc=${broken_result.rc})
    END

    # --- Step 5: Execute all plans (strategize + execute phases) ---
    @{executed_ids}=    Execute Batch Plans    @{plan_ids}
    ${executed_count}=    Get Length    ${executed_ids}
    ${total_plan_count_at_execute}=    Get Length    ${plan_ids}
    Log    Plans that completed execution: ${executed_count} / ${total_plan_count_at_execute}

    # --- Step 6: Apply successfully-executed plans ---
    @{applied_ids}=    Apply Batch Plans    @{executed_ids}
    ${applied_count}=    Get Length    ${applied_ids}
    Log    Plans that reached applied state: ${applied_count} / ${executed_count}

    # --- Step 7: Verify batch results via plan list ---
    # 7a: Unfiltered listing — smoke test and verify healthy plan IDs appear
    ${list_result}=    Run CleverAgents Command
    ...    plan    list    --format    plain
    ...    timeout=30s
    Log    Final plan list stdout: ${list_result.stdout}
    Log    Final plan list stderr: ${list_result.stderr}
    Should Not Contain    ${list_result.stderr}    Traceback
    FOR    ${pid}    IN    @{applied_ids}
        Should Contain    ${list_result.stdout}    ${pid}
        ...    Applied plan ID ${pid} not found in plan list output
    END

    # 7b: Filtered listing — verify --state applied returns successful plans
    ${applied_list}=    Run CleverAgents Command
    ...    plan    list    --state    applied    --format    plain
    ...    timeout=30s
    Log    Applied plan list stdout: ${applied_list.stdout}
    ${applied_matches}=    Get Regexp Matches    ${applied_list.stdout}
    ...    processing_state:\\s*applied
    ${success_count}=    Get Length    ${applied_matches}
    Log    Plans in 'applied' state: ${success_count}
    # At least 2 of 3 healthy packages must reach 'applied' state.
    # Using >= 2 (not == 3) because LLM-generated changes can occasionally fail
    # during apply (e.g. merge conflicts, empty changesets from the LLM producing
    # no edits).  Requiring >= 2 verifies the batch mechanism works while
    # tolerating one transient apply failure.
    Should Be True    ${success_count} >= 2
    ...    Expected at least 2 plans (of 3 healthy) to reach 'applied' state but found ${success_count}

    # --- Step 8: Verify error handling for the broken action ---
    # Count how many plans failed during execution (didn't make it to executed_ids)
    ${total_plan_count}=    Get Length    ${plan_ids}
    ${failed_execution_count}=    Evaluate    ${total_plan_count} - ${executed_count}
    Log    Plans that failed during execution: ${failed_execution_count}
    # Also check for errored plans via --state errored filter
    ${errored_list}=    Run CleverAgents Command
    ...    plan    list    --state    errored    --format    plain
    ...    timeout=30s
    Log    Errored plan list stdout: ${errored_list.stdout}
    ${errored_matches}=    Get Regexp Matches    ${errored_list.stdout}
    ...    processing_state:\\s*errored
    ${error_count}=    Get Length    ${errored_matches}
    Log    Plans in 'errored' state: ${error_count}
    # The broken action (non-existent actor) should cause at least one failure.
    # Error handling is demonstrated if ANY of the following hold:
    #   (a) broken action's plan use failed (rc != 0)
    #   (b) the broken plan's ID is NOT in executed_ids (failed during execution)
    #   (c) the broken plan's ID appears in the errored list
    IF    ${broken_plan_failed_at_creation}
        Log    Error handling demonstrated: broken plan failed at creation (rc != 0)
    ELSE IF    "${broken_plan_id}" != "${EMPTY}"
        # Verify the specific broken plan ID either did NOT complete execution
        # or appears in the errored list
        ${broken_in_executed}=    Evaluate    """${broken_plan_id}""" in ${executed_ids}
        ${broken_in_errored}=    Evaluate    """${broken_plan_id}""" in """${errored_list.stdout}"""
        ${broken_failed}=    Evaluate    not ${broken_in_executed} or ${broken_in_errored}
        Should Be True    ${broken_failed}
        ...    Broken plan ${broken_plan_id} should have failed but was found in executed_ids and not in errored list
        Log    Error handling demonstrated: broken plan ${broken_plan_id} failed (not in executed=${broken_in_executed}, in errored=${broken_in_errored})
    ELSE
        # Fallback: general failure count check
        ${broken_demonstrated}=    Evaluate
        ...    ${error_count} >= 1 or ${failed_execution_count} >= 1
        Should Be True    ${broken_demonstrated}
        ...    Error handling not demonstrated: 0 errored, 0 failed execution
    END