diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d2eb97cf..d8a83fd6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- Added E2E Robot Framework test for Specification Workflow Example 8: Cloud + Infrastructure Management (supervised profile). Exercises custom resource + type registration (`local/terraform-state`), custom skill with tools and + skill composition (`includes: [local/file-ops]`), stub validation + registration, supervised automation profile verification, and full plan + lifecycle (`plan use` → `plan execute` → `plan diff` → `plan lifecycle-apply`) + with post-apply git commit and plan state assertions. (#754) - Added TDD bug-capture tests for bug #1076 — `use_action()` does not propagate `automation_profile` to Plan. Three Behave BDD scenarios (`@tdd_bug @tdd_bug_1076 @tdd_expected_fail`) verify the full precedence diff --git a/robot/e2e/wf08_cloud_infra.robot b/robot/e2e/wf08_cloud_infra.robot new file mode 100644 index 000000000..91c5a185c --- /dev/null +++ b/robot/e2e/wf08_cloud_infra.robot @@ -0,0 +1,677 @@ +*** Settings *** +Documentation E2E workflow example 8 — Cloud Infrastructure Management. +... +... Scenario: A DevOps team uses CleverAgents with the +... **supervised** automation profile to analyse Terraform-managed +... infrastructure, identify unused or over-provisioned resources, +... and generate cost-optimisation recommendations. +... +... The test registers a custom ``local/terraform-state`` resource +... type, creates a ``local/terraform-ops`` skill with three custom +... tools and skill composition (``includes: [local/file-ops]``), +... registers stub validations (tf-validate, tf-plan) and attaches +... them to the project, creates an infrastructure-analysis action +... with invariants, exercises the custom resource type, and drives +... the full plan lifecycle +... (strategize → execute → diff → lifecycle-apply). +... +... **Zero mocking** — real CLI, real LLM API keys. +... +... Expected aggregate worst-case runtime: ~18 min (five +... LLM-interacting commands with varying timeouts: 120–300 s). +... Typical runtime with fast providers: 2–5 min. +Resource common_e2e.resource +Suite Setup WF08 Suite Setup +Suite Teardown E2E Suite Teardown +Force Tags E2E + +*** Variables *** +${TF_REPO_NAME} terraform-infra +${ACTION_NAME} local/infra-analyze +${PROJECT_NAME} local/infra-project +${RESOURCE_NAME} local/tf-infra +${TF_RES_NAME} local/tf-state-res +${WF08_TF_VAL} local/tf-validate +${WF08_TF_PLAN_VAL} local/tf-plan + +# ── Terraform fixture files ───────────────────────────────────── +${MAIN_TF} SEPARATOR=\n +... terraform { +... ${SPACE}${SPACE}required_providers { +... ${SPACE}${SPACE}${SPACE}${SPACE}aws = { +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}source = "hashicorp/aws" +... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}version = "~> 5.0" +... ${SPACE}${SPACE}${SPACE}${SPACE}} +... ${SPACE}${SPACE}} +... ${SPACE}${SPACE}required_version = ">= 1.5.0" +... } +... +... provider "aws" { +... ${SPACE}${SPACE}region = var.aws_region +... } +... +... resource "aws_instance" "web" { +... ${SPACE}${SPACE}count = var.instance_count +... ${SPACE}${SPACE}ami = var.ami_id +... ${SPACE}${SPACE}instance_type = var.instance_type +... +... ${SPACE}${SPACE}tags = { +... ${SPACE}${SPACE}${SPACE}${SPACE}Name = "web-\${count.index}" +... ${SPACE}${SPACE}${SPACE}${SPACE}Environment = var.environment +... ${SPACE}${SPACE}${SPACE}${SPACE}ManagedBy = "terraform" +... ${SPACE}${SPACE}} +... } +... +... resource "aws_s3_bucket" "logs" { +... ${SPACE}${SPACE}bucket = "\${var.project_name}-logs-\${var.environment}" +... +... ${SPACE}${SPACE}tags = { +... ${SPACE}${SPACE}${SPACE}${SPACE}Environment = var.environment +... ${SPACE}${SPACE}${SPACE}${SPACE}Purpose = "application-logs" +... ${SPACE}${SPACE}} +... } +... +... resource "aws_s3_bucket" "backups" { +... ${SPACE}${SPACE}bucket = "\${var.project_name}-backups-\${var.environment}" +... +... ${SPACE}${SPACE}tags = { +... ${SPACE}${SPACE}${SPACE}${SPACE}Environment = var.environment +... ${SPACE}${SPACE}${SPACE}${SPACE}Purpose = "database-backups" +... ${SPACE}${SPACE}} +... } + +${VARIABLES_TF} SEPARATOR=\n +... variable "aws_region" { +... ${SPACE}${SPACE}description = "AWS region for resources" +... ${SPACE}${SPACE}type = string +... ${SPACE}${SPACE}default = "us-east-1" +... } +... +... variable "environment" { +... ${SPACE}${SPACE}description = "Deployment environment" +... ${SPACE}${SPACE}type = string +... ${SPACE}${SPACE}default = "staging" +... } +... +... variable "instance_count" { +... ${SPACE}${SPACE}description = "Number of EC2 instances" +... ${SPACE}${SPACE}type = number +... ${SPACE}${SPACE}default = 3 +... } +... +... variable "instance_type" { +... ${SPACE}${SPACE}description = "EC2 instance type" +... ${SPACE}${SPACE}type = string +... ${SPACE}${SPACE}default = "t3.xlarge" +... } +... +... variable "ami_id" { +... ${SPACE}${SPACE}description = "AMI ID for EC2 instances" +... ${SPACE}${SPACE}type = string +... ${SPACE}${SPACE}default = "ami-0c55b159cbfafe1f0" +... } +... +... variable "project_name" { +... ${SPACE}${SPACE}description = "Project identifier for resource naming" +... ${SPACE}${SPACE}type = string +... ${SPACE}${SPACE}default = "acme-platform" +... } + +${OUTPUTS_TF} SEPARATOR=\n +... output "instance_ids" { +... ${SPACE}${SPACE}description = "IDs of the web EC2 instances" +... ${SPACE}${SPACE}value = aws_instance.web[*].id +... } +... +... output "log_bucket_arn" { +... ${SPACE}${SPACE}description = "ARN of the logs S3 bucket" +... ${SPACE}${SPACE}value = aws_s3_bucket.logs.arn +... } +... +... output "backup_bucket_arn" { +... ${SPACE}${SPACE}description = "ARN of the backups S3 bucket" +... ${SPACE}${SPACE}value = aws_s3_bucket.backups.arn +... } + +# ── Resource type YAML ────────────────────────────────────────── +# NOTE: Spec divergences documented here: +# - sandbox_strategy: The spec lists filesystem_copy. The test uses +# copy_on_write because the implementation maps both values identically +# (copy_on_write is the canonical enum variant). +# - resource_kind: The spec uses "physical: true"; the implementation uses +# "resource_kind: physical" (enum field). +# - handler/child_types/workspace: The spec includes these fields but the +# implementation's ResourceType schema does not require them for +# user-addable types. +${RESOURCE_TYPE_YAML} SEPARATOR=\n +... name: local/terraform-state +... description: "Terraform state and configuration directory" +... resource_kind: physical +... sandbox_strategy: copy_on_write +... user_addable: true +... cli_args: +... ${SPACE}${SPACE}- name: path +... ${SPACE}${SPACE}${SPACE}${SPACE}type: path +... ${SPACE}${SPACE}${SPACE}${SPACE}required: true +... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Path to the Terraform project directory" +... capabilities: +... ${SPACE}${SPACE}read: true +... ${SPACE}${SPACE}write: true +... ${SPACE}${SPACE}sandbox: true +... ${SPACE}${SPACE}checkpoint: false + +# ── Terraform-ops skill YAML (AC #3: custom skill + tools + include_skills) ── +# Spec divergences: +# - Tools use namespaced refs (SkillToolRefSchema); input_schema is not +# supported on tool refs (only on inline_tools). The spec's input_schema +# is adapted to tool-ref-compatible fields (name, description, writes, +# checkpointable). +# - The spec uses "include_skills"; the implementation schema uses "includes". +# - Tool names are namespaced (local/terraform_plan vs spec's bare +# terraform_plan). +${SKILL_YAML} SEPARATOR=\n +... name: local/terraform-ops +... description: "Terraform infrastructure operations" +... tools: +... ${SPACE}${SPACE}- name: local/terraform_plan +... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Run terraform plan and return the execution plan" +... ${SPACE}${SPACE}${SPACE}${SPACE}writes: false +... ${SPACE}${SPACE}${SPACE}${SPACE}checkpointable: false +... ${SPACE}${SPACE}- name: local/terraform_show +... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Show current state of a Terraform resource" +... ${SPACE}${SPACE}${SPACE}${SPACE}writes: false +... ${SPACE}${SPACE}${SPACE}${SPACE}checkpointable: false +... ${SPACE}${SPACE}- name: local/cloud_metrics +... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Fetch CloudWatch/cloud monitoring metrics for a resource" +... ${SPACE}${SPACE}${SPACE}${SPACE}writes: false +... ${SPACE}${SPACE}${SPACE}${SPACE}checkpointable: false +... includes: +... ${SPACE}${SPACE}- name: local/file-ops + +*** Test Cases *** +WF08 Cloud Infrastructure Management Supervised Profile + [Documentation] Full lifecycle: create Terraform repo, register custom + ... resource type, create terraform-ops skill with tools and + ... skill composition, register stub validations, create action + ... with invariants, register resources (including custom + ... terraform-state type), create project with validations + ... attached, plan use with supervised profile, execute + ... (strategize + execute), diff, lifecycle-apply, and verify + ... infrastructure analysis output. + [Timeout] 30 minutes + [Teardown] WF08 Test Teardown + + # Guard: skip gracefully when no LLM API keys are available. + Skip If No LLM Keys + + # Initialise test variable for teardown access. + Set Test Variable ${WF08_PLAN_ID} ${EMPTY} + + # Generate a unique suffix to avoid UNIQUE constraint collisions on + # repeated E2E runs (parallel CI safety). + ${suffix}= Evaluate __import__('uuid').uuid4().hex[:12] + + # Pick actor based on available API key (Anthropic preferred). + ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) + IF ${has_anthropic} + ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 + ELSE + ${actor}= Set Variable openai/gpt-4o + END + + # ------------------------------------------------------------------ + # 1. Create a temporary git repo with Terraform configuration files + # ------------------------------------------------------------------ + ${repo_dir}= Create Temp Git Repo ${TF_REPO_NAME}-${suffix} + + Create File ${repo_dir}${/}main.tf ${MAIN_TF} + Create File ${repo_dir}${/}variables.tf ${VARIABLES_TF} + Create File ${repo_dir}${/}outputs.tf ${OUTPUTS_TF} + + ${git_add}= Run Process git add . cwd=${repo_dir} + ... timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 + ... git add failed: ${git_add.stderr} + ${git_commit}= Run Process git commit -m Add Terraform configuration + ... cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 + ... git commit failed: ${git_commit.stderr} + + # ------------------------------------------------------------------ + # 2. Register a custom resource type (local/terraform-state) + # ------------------------------------------------------------------ + ${rt_yaml}= Set Variable ${SUITE_HOME}${/}terraform-state-type.yaml + Create File ${rt_yaml} ${RESOURCE_TYPE_YAML} + + ${rt_result}= Run CleverAgents Command + ... resource type add --config ${rt_yaml} --format json + Should Be Equal As Integers ${rt_result.rc} 0 + ... Resource type registration failed: ${rt_result.stderr} + Should Not Contain ${rt_result.stdout}${rt_result.stderr} Traceback + Should Not Contain ${rt_result.stdout}${rt_result.stderr} INTERNAL + Output Should Contain ${rt_result} local/terraform-state + Log Resource type registration: rc=${rt_result.rc} + + # ------------------------------------------------------------------ + # 3. Register custom terraform-ops skill (AC #3: tools + includes) + # ------------------------------------------------------------------ + ${skill_yaml_path}= Set Variable ${SUITE_HOME}${/}terraform-ops.yaml + Create File ${skill_yaml_path} ${SKILL_YAML} + + ${skill_result}= Run CleverAgents Command + ... skill add --config ${skill_yaml_path} --format json + Should Be Equal As Integers ${skill_result.rc} 0 + ... Skill registration failed: ${skill_result.stderr} + Should Not Contain ${skill_result.stdout}${skill_result.stderr} Traceback + Should Not Contain ${skill_result.stdout}${skill_result.stderr} INTERNAL + Output Should Contain ${skill_result} local/terraform-ops + # AC #3: Verify skill composition — includes field processed local/file-ops + Output Should Contain ${skill_result} local/file-ops + Log Skill registration: rc=${skill_result.rc} + + # ------------------------------------------------------------------ + # 3b. Register stub validations (spec WF8 Step 2: tf-validate, tf-plan) + # Uses inline code: blocks (WF07 pattern) to exercise the + # registration path without requiring external terraform tooling. + # ------------------------------------------------------------------ + ${tf_val_yaml}= Catenate SEPARATOR=\n + ... name: ${WF08_TF_VAL} + ... description: Terraform validate stub for WF08 E2E + ... source: custom + ... mode: required + ... code: | + ... ${SPACE}${SPACE}def run(inputs): + ... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"passed": True, "message": "terraform validate passed"} + ${tf_val_path}= Set Variable ${SUITE_HOME}${/}wf08-tf-validate.yaml + Create File ${tf_val_path} ${tf_val_yaml} + ${tf_val_result}= Run CleverAgents Command + ... validation add --config ${tf_val_path} --format json expected_rc=None + Should Not Contain ${tf_val_result.stdout}${tf_val_result.stderr} Traceback + Should Not Contain ${tf_val_result.stdout}${tf_val_result.stderr} INTERNAL + IF ${tf_val_result.rc} != 0 + Log Validation tf-validate registration returned rc=${tf_val_result.rc} WARN + END + Log Validation tf-validate registration: rc=${tf_val_result.rc} + + # NOTE: tf-plan uses "informational" mode per spec WF8 Step 2 — the + # spec intentionally distinguishes a blocking validation (tf-validate, + # mode: required) from an advisory one (tf-plan, mode: informational). + ${tf_plan_yaml}= Catenate SEPARATOR=\n + ... name: ${WF08_TF_PLAN_VAL} + ... description: Terraform plan stub for WF08 E2E + ... source: custom + ... mode: informational + ... code: | + ... ${SPACE}${SPACE}def run(inputs): + ... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}return {"passed": True, "message": "terraform plan passed"} + ${tf_plan_path}= Set Variable ${SUITE_HOME}${/}wf08-tf-plan.yaml + Create File ${tf_plan_path} ${tf_plan_yaml} + ${tf_plan_result}= Run CleverAgents Command + ... validation add --config ${tf_plan_path} --format json expected_rc=None + Should Not Contain ${tf_plan_result.stdout}${tf_plan_result.stderr} Traceback + Should Not Contain ${tf_plan_result.stdout}${tf_plan_result.stderr} INTERNAL + IF ${tf_plan_result.rc} != 0 + Log Validation tf-plan registration returned rc=${tf_plan_result.rc} WARN + END + Log Validation tf-plan registration: rc=${tf_plan_result.rc} + + # ------------------------------------------------------------------ + # 4. Create the infrastructure-analysis action YAML (dynamic actor) + # ------------------------------------------------------------------ + # Spec divergence: The spec lists 3 invariants; this test uses 4 for + # broader coverage. The spec uses "production_safety" as one of the + # invariants; this test uses more descriptive text equivalents. + ${action_yaml_content}= Catenate SEPARATOR=\n + ... name: ${ACTION_NAME} + ... description: "Analyse Terraform infrastructure for cost optimisation" + ... long_description: | + ... ${SPACE}${SPACE}Analyse Terraform-managed cloud infrastructure to identify + ... ${SPACE}${SPACE}unused or over-provisioned resources and generate + ... ${SPACE}${SPACE}cost-optimisation recommendations. + ... strategy_actor: ${actor} + ... execution_actor: ${actor} + ... definition_of_done: | + ... ${SPACE}${SPACE}Infrastructure analysis is complete with actionable + ... ${SPACE}${SPACE}cost-optimisation recommendations documented. + ... reusable: true + ... read_only: false + ... automation_profile: supervised + ... invariants: + ... ${SPACE}${SPACE}- "Never destroy stateful resources (databases, S3 buckets) without verified backup" + ... ${SPACE}${SPACE}- "All proposed changes must be Terraform plan-safe (no forced replacements)" + ... ${SPACE}${SPACE}- "Maintain minimum instance count for high-availability services" + ... ${SPACE}${SPACE}- "Do not modify resources tagged Environment=production" + ${action_yaml}= Set Variable ${SUITE_HOME}${/}infra-analyze.yaml + Create File ${action_yaml} ${action_yaml_content} + + ${action_result}= Run CleverAgents Command + ... action create --config ${action_yaml} --format json + Should Be Equal As Integers ${action_result.rc} 0 + ... Action create failed: ${action_result.stderr} + Should Not Contain ${action_result.stdout}${action_result.stderr} Traceback + Should Not Contain ${action_result.stdout}${action_result.stderr} INTERNAL + Output Should Contain ${action_result} ${ACTION_NAME} + Log Action create: rc=${action_result.rc} + + # ------------------------------------------------------------------ + # 5. Register resources — git-checkout AND custom terraform-state type + # ------------------------------------------------------------------ + # 5a. git-checkout resource + ${res_name}= Set Variable ${RESOURCE_NAME}-${suffix} + ${res_result}= Run CleverAgents Command + ... resource add git-checkout ${res_name} + ... --path ${repo_dir} + ... --description Terraform infrastructure repository + Should Be Equal As Integers ${res_result.rc} 0 + ... Resource add (git-checkout) failed: ${res_result.stderr} + Should Not Contain ${res_result.stdout}${res_result.stderr} Traceback + Should Not Contain ${res_result.stdout}${res_result.stderr} INTERNAL + Log Resource add (git-checkout): rc=${res_result.rc} + + # 5b. Exercise the custom terraform-state resource type (milestone 6 + # custom resource type pattern) + ${tf_res_name}= Set Variable ${TF_RES_NAME}-${suffix} + ${tf_res_result}= Run CleverAgents Command + ... resource add local/terraform-state ${tf_res_name} + ... --path ${repo_dir} + Should Be Equal As Integers ${tf_res_result.rc} 0 + ... Resource add (terraform-state) failed: ${tf_res_result.stderr} + Should Not Contain ${tf_res_result.stdout}${tf_res_result.stderr} Traceback + Should Not Contain ${tf_res_result.stdout}${tf_res_result.stderr} INTERNAL + Log Resource add (terraform-state): rc=${tf_res_result.rc} + + # ------------------------------------------------------------------ + # 6. Create project with invariants, linking both resources + # ------------------------------------------------------------------ + ${proj_name}= Set Variable ${PROJECT_NAME}-${suffix} + ${proj_result}= Run CleverAgents Command + ... project create ${proj_name} + ... --description Cloud infrastructure cost-optimisation project + ... --resource ${res_name} + ... --resource ${tf_res_name} + ... --invariant Never destroy stateful resources without backup + ... --invariant All changes must be Terraform plan-safe + Should Be Equal As Integers ${proj_result.rc} 0 + ... Project create failed: ${proj_result.stderr} + Should Not Contain ${proj_result.stdout}${proj_result.stderr} Traceback + Should Not Contain ${proj_result.stdout}${proj_result.stderr} INTERNAL + Log Project create: rc=${proj_result.rc} + + # 6b. Attach validations to project (spec WF8 Step 2) + ${attach_val_result}= Run CleverAgents Command + ... validation attach --format json + ... --project ${proj_name} ${res_name} ${WF08_TF_VAL} + ... expected_rc=None + Log Validation attach (tf-validate): rc=${attach_val_result.rc} + Should Not Contain ${attach_val_result.stdout}${attach_val_result.stderr} Traceback + Should Not Contain ${attach_val_result.stdout}${attach_val_result.stderr} INTERNAL + ${attach_plan_result}= Run CleverAgents Command + ... validation attach --format json + ... --project ${proj_name} ${res_name} ${WF08_TF_PLAN_VAL} + ... expected_rc=None + Log Validation attach (tf-plan): rc=${attach_plan_result.rc} + Should Not Contain ${attach_plan_result.stdout}${attach_plan_result.stderr} Traceback + Should Not Contain ${attach_plan_result.stdout}${attach_plan_result.stderr} INTERNAL + + # ------------------------------------------------------------------ + # 7. Plan use with supervised automation profile (--format json) + # ------------------------------------------------------------------ + # Spec divergence: WF8 Step 3 shows --arg optimization_targets="compute,storage" + # --arg min_savings_threshold=25.0. Omitted here for simplification — the LLM + # receives infrastructure context from resources and performs analysis without + # explicit arg overrides. + ${plan_result}= Run CleverAgents Command + ... plan use + ... --automation-profile supervised + ... --format json + ... ${ACTION_NAME} ${proj_name} + ... expected_rc=None timeout=180s + Should Be Equal As Integers ${plan_result.rc} 0 + ... plan use failed (rc=${plan_result.rc}): ${plan_result.stderr} + Should Not Contain ${plan_result.stdout}${plan_result.stderr} Traceback + Should Not Contain ${plan_result.stdout}${plan_result.stderr} INTERNAL + Log Plan use: rc=${plan_result.rc} + + ${plan_id}= Safe Parse Json Field ${plan_result.stdout} plan_id + Should Not Be Empty ${plan_id} Could not extract plan_id from plan use output + Set Test Variable ${WF08_PLAN_ID} ${plan_id} + Log Extracted plan_id: ${plan_id} + + # AC #5: Verify supervised automation profile was applied. + ${resolved_profile}= Safe Parse Json Field ${plan_result.stdout} automation_profile + ${resolved_profile}= Set Variable If $resolved_profile is None ${EMPTY} ${resolved_profile} + ${resolved_profile}= Set Variable If '${resolved_profile}' == 'None' ${EMPTY} ${resolved_profile} + IF '${resolved_profile}' == '' + ${r_profile_status}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=120s + Should Be Equal As Integers ${r_profile_status.rc} 0 + ... plan status for profile verification failed (rc=${r_profile_status.rc}): ${r_profile_status.stderr} + Should Not Contain ${r_profile_status.stdout}${r_profile_status.stderr} Traceback + Should Not Contain ${r_profile_status.stdout}${r_profile_status.stderr} INTERNAL + ${resolved_profile}= Safe Parse Json Field ${r_profile_status.stdout} automation_profile + ${resolved_profile}= Set Variable If $resolved_profile is None ${EMPTY} ${resolved_profile} + ${resolved_profile}= Set Variable If '${resolved_profile}' == 'None' ${EMPTY} ${resolved_profile} + END + Should Be Equal As Strings ${resolved_profile} supervised + ... Expected automation_profile 'supervised' but got '${resolved_profile}' + + # ------------------------------------------------------------------ + # 8. Plan execute — strategize phase + # ------------------------------------------------------------------ + ${strat_result}= Run CleverAgents Command + ... plan execute ${plan_id} + ... --format plain + ... expected_rc=None timeout=300s + Log Plan execute (strategize): rc=${strat_result.rc} + Should Not Contain ${strat_result.stdout}${strat_result.stderr} Traceback + ... Plan execute (strategize) produced a Python traceback + Should Not Contain ${strat_result.stdout}${strat_result.stderr} INTERNAL + IF ${strat_result.rc} != 0 + Fail plan execute (strategize) failed (rc=${strat_result.rc}): ${strat_result.stderr} + END + + # Intermediate plan status check after strategize — verify phase + # transition before proceeding (mirrors WF05's lifecycle pattern). + ${mid_status_result}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=120s + Should Be Equal As Integers ${mid_status_result.rc} 0 + ... plan status (post-strategize) failed (rc=${mid_status_result.rc}): ${mid_status_result.stderr} + Should Not Contain ${mid_status_result.stdout}${mid_status_result.stderr} Traceback + Should Not Contain ${mid_status_result.stdout}${mid_status_result.stderr} INTERNAL + Log Plan status (post-strategize): ${mid_status_result.stdout} + + # ------------------------------------------------------------------ + # 9. Plan execute — execute phase + # ------------------------------------------------------------------ + ${exec_result}= Run CleverAgents Command + ... plan execute ${plan_id} + ... --format plain + ... expected_rc=None timeout=300s + Log Plan execute (execute): rc=${exec_result.rc} + Should Not Contain ${exec_result.stdout}${exec_result.stderr} Traceback + ... Plan execute (execute) produced a Python traceback + Should Not Contain ${exec_result.stdout}${exec_result.stderr} INTERNAL + IF ${exec_result.rc} != 0 + Fail plan execute (execute) failed (rc=${exec_result.rc}): ${exec_result.stderr} + END + + # ------------------------------------------------------------------ + # 10. Plan diff + # ------------------------------------------------------------------ + ${diff_result}= Run CleverAgents Command + ... plan diff ${plan_id} + ... --format plain + ... expected_rc=None timeout=120s + Log Plan diff: rc=${diff_result.rc} + Should Not Contain ${diff_result.stdout}${diff_result.stderr} Traceback + ... Plan diff produced a Python traceback + Should Not Contain ${diff_result.stdout}${diff_result.stderr} INTERNAL + IF ${diff_result.rc} != 0 + Fail plan diff failed (rc=${diff_result.rc}): ${diff_result.stderr} + END + # Verify diff output contains meaningful infrastructure-related content. + Should Not Be Empty ${diff_result.stdout} Plan diff produced no output + ${diff_lower}= Evaluate ($diff_result.stdout).lower() + ${has_diff_signal}= Evaluate 'terraform' in $diff_lower or 'infrastructure' in $diff_lower or 'diff' in $diff_lower or 'change' in $diff_lower or 'file' in $diff_lower or 'instance' in $diff_lower or 'bucket' in $diff_lower + Should Be True ${has_diff_signal} + ... plan diff output should include meaningful diff/change indicators + + # ------------------------------------------------------------------ + # 11. Plan lifecycle-apply --yes + # ------------------------------------------------------------------ + # Capture baseline SHA before apply for accurate commit verification. + ${baseline_sha_result}= Run Process git rev-parse HEAD + ... cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${baseline_sha_result.rc} 0 + ... git rev-parse HEAD failed: ${baseline_sha_result.stderr} + ${baseline_sha}= Strip String ${baseline_sha_result.stdout} + Log Baseline SHA before lifecycle-apply: ${baseline_sha} + + ${apply_result}= Run CleverAgents Command + ... plan lifecycle-apply --yes ${plan_id} + ... --format plain + ... expected_rc=None timeout=180s + Log Plan lifecycle-apply: rc=${apply_result.rc} + Should Not Contain ${apply_result.stdout}${apply_result.stderr} Traceback + ... Plan lifecycle-apply produced a Python traceback + Should Not Contain ${apply_result.stdout}${apply_result.stderr} INTERNAL + IF ${apply_result.rc} != 0 + Fail plan lifecycle-apply failed (rc=${apply_result.rc}): ${apply_result.stderr} + END + + # ------------------------------------------------------------------ + # 12. Verify plan state after lifecycle-apply + # ------------------------------------------------------------------ + ${r_status_after_apply}= Run CleverAgents Command + ... plan status ${plan_id} --format json + ... expected_rc=None timeout=120s + Should Be Equal As Integers ${r_status_after_apply.rc} 0 + ... plan status after lifecycle-apply failed (rc=${r_status_after_apply.rc}): ${r_status_after_apply.stderr} + Should Not Contain ${r_status_after_apply.stdout}${r_status_after_apply.stderr} Traceback + Should Not Contain ${r_status_after_apply.stdout}${r_status_after_apply.stderr} INTERNAL + ${apply_phase}= Safe Parse Json Field ${r_status_after_apply.stdout} phase + ${apply_state}= Safe Parse Json Field ${r_status_after_apply.stdout} processing_state + Should Not Be Empty ${apply_phase} plan status after apply should include phase + Should Not Be Empty ${apply_state} plan status after apply should include processing_state + ${apply_phase_lower}= Evaluate ($apply_phase).lower() + IF 'apply' not in $apply_phase_lower + Log Post-apply phase is '${apply_phase}' instead of apply; treating terminal state as authoritative WARN + END + ${is_terminal_state}= Evaluate ($apply_state.lower() in ['applied', 'constrained', 'errored', 'cancelled', 'complete']) + ${is_apply_progress_state}= Evaluate ('apply' in $apply_phase_lower) + Should Be True ${is_terminal_state} or ${is_apply_progress_state} + ... expected lifecycle-apply to produce terminal state or apply-phase progress (phase=${apply_phase}, state=${apply_state}) + + # ------------------------------------------------------------------ + # 13. Post-apply verification — check target repo for new commits + # ------------------------------------------------------------------ + ${post_sha_result}= Run Process git rev-parse HEAD + ... cwd=${repo_dir} timeout=60s on_timeout=kill + Should Be Equal As Integers ${post_sha_result.rc} 0 + ... git rev-parse HEAD (post-apply) failed: ${post_sha_result.stderr} + ${post_sha}= Strip String ${post_sha_result.stdout} + Log Post-apply SHA: ${post_sha} + # Hard-assert baseline SHA was captured and the repo is valid. + # The LLM may or may not produce new commits (non-deterministic), so + # a SHA mismatch is logged as a warning rather than a hard failure — + # matching the WF05 resilience pattern. + IF '${baseline_sha}' == '${post_sha}' + Log No new commits from lifecycle-apply — LLM may not have produced file changes in this run WARN + ELSE + Log Lifecycle-apply produced new commits (baseline=${baseline_sha} -> post=${post_sha}) + END + + # ------------------------------------------------------------------ + # 14. Verify infrastructure analysis output + # ------------------------------------------------------------------ + # NOTE: plan_result (from ``plan use --format json``) is deliberately + # excluded to prevent action metadata / description fields from + # trivially satisfying the analysis keyword thresholds. + ${all_output}= Collect All Output + ... ${strat_result} ${exec_result} + ... ${diff_result} ${apply_result} + + Verify Infrastructure Analysis ${all_output} + +*** Keywords *** +WF08 Suite Setup + [Documentation] E2E Suite Setup plus database initialisation. + E2E Suite Setup + ${init}= Run CleverAgents Command init --force --yes + Should Be Equal As Integers ${init.rc} 0 + +WF08 Test Teardown + [Documentation] Log diagnostic context on failure for debugging. + ${plan_id}= Get Variable Value ${WF08_PLAN_ID} ${EMPTY} + IF '${plan_id}' != '' + ${status} ${result}= Run Keyword And Ignore Error + ... Run CleverAgents Command plan status ${plan_id} --format json expected_rc=None timeout=30s + IF '${status}' == 'PASS' + Log Teardown plan status: ${result.stdout} WARN + END + ${tree_status} ${tree_result}= Run Keyword And Ignore Error + ... Run CleverAgents Command plan tree ${plan_id} --format json expected_rc=None timeout=30s + IF '${tree_status}' == 'PASS' + Log Teardown plan tree: ${tree_result.stdout} WARN + END + END + +Collect All Output + [Documentation] Concatenate stdout and stderr from multiple results. + [Arguments] @{results} + ${combined}= Set Variable ${EMPTY} + FOR ${r} IN @{results} + ${combined}= Catenate SEPARATOR=\n ${combined} ${r.stdout} ${r.stderr} + END + RETURN ${combined} + +Verify Infrastructure Analysis + [Documentation] Flexible verification that the plan lifecycle produced + ... infrastructure-related analysis output. Requires at + ... least 5 keyword matches across two categories for + ... resilience against generic CLI noise: a broad set of + ... infrastructure terms (some may echo from fixture data) + ... and an analysis-specific set that indicates actual LLM + ... reasoning. Asserts no Python tracebacks appear. + [Arguments] ${output} + ${lower}= Evaluate $output.lower() + + # Negative assertions: no tracebacks or unhandled errors + Should Not Contain ${lower} traceback (most recent call last) + ... Infrastructure analysis output contains a Python traceback + + # Broad infrastructure terms — may overlap with fixture data / CLI echoes. + ${broad_count}= Set Variable ${0} + FOR ${term} IN + ... terraform infrastructure aws provider + ... instance_type s3_bucket ec2 bucket + ... t3.xlarge instance_count + ${contains}= Evaluate "${term}" in $lower + IF ${contains} + ${broad_count}= Evaluate ${broad_count} + 1 + END + END + + # Analysis-specific terms — these should only appear from actual LLM + # analysis, not from CLI argument echoes or fixture data. + # NOTE: "optimi" and "optimis" were merged into single "optimi" to avoid + # double-counting a single mention of "optimisation" / "optimization". + # NOTE: "provision" was removed — it is a substring of "over-provision" + # and would inflate analysis_count by 1 for a single mention. + ${analysis_count}= Set Variable ${0} + FOR ${term} IN + ... optimi cost savings right-siz + ... unused over-provision recommend + ${contains}= Evaluate "${term}" in $lower + IF ${contains} + ${analysis_count}= Evaluate ${analysis_count} + 1 + END + END + + ${total_count}= Evaluate ${broad_count} + ${analysis_count} + Should Be True ${total_count} >= 5 + ... Expected at least 5 infrastructure-specific keywords in combined output but found ${total_count} (broad=${broad_count}, analysis=${analysis_count}). + Should Be True ${analysis_count} >= 2 + ... Expected at least 2 analysis-specific keywords (cost, optimi, savings, right-siz, unused, over-provision, recommend) but found ${analysis_count}.