diff --git a/CHANGELOG.md b/CHANGELOG.md index 36cb64469..9d1af118c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Added TDD bug-capture E2E tests for bug #1028 — ACMS indexing pipeline not + wired into CLI. Four Robot Framework E2E tests prove ContextTierService starts + empty on every CLI invocation. Tests use ``@tdd_expected_fail`` until the bug + fix is merged. (#1029) - Added Fix-then-Revalidate orchestration loop for required validations: bounded retry with configurable limits (0--100 per Safety Profile), strategy revision escalation via ``auto_strategy_revision`` float diff --git a/noxfile.py b/noxfile.py index bc9365c4f..5e78c9b80 100644 --- a/noxfile.py +++ b/noxfile.py @@ -539,7 +539,7 @@ def serve_docs(session: nox.Session): @nox.session(python=SUPPORTED_PYTHONS, reuse_venv=True, venv_backend="uv") def build(session: nox.Session): """Build the wheel distribution.""" - session.install("build") + session.install("build", "pip") session.run("python", "-m", "build", "--wheel") diff --git a/robot/e2e/common_e2e.resource b/robot/e2e/common_e2e.resource index b36aac3c9..8008931d2 100644 --- a/robot/e2e/common_e2e.resource +++ b/robot/e2e/common_e2e.resource @@ -137,6 +137,96 @@ Safe Parse Json Field Log Safe Parse Json Field: JSON parse failed for field '${field_name}'. Strategy 1: ${strategy1_err}; Strategy 2: ${strategy2_err} WARN RETURN ${EMPTY} +Run CLI + [Documentation] Run ``python -m cleveragents`` inside a workspace directory + ... with the correct environment variables. + ... + ... Requires the calling suite to set ``${WS}`` (workspace + ... directory) and ``${SUITE_HOME}`` (CLEVERAGENTS_HOME) as + ... suite variables before calling this keyword. + ... + ... Do not embed raw stdout/stderr in assertion failure messages — + ... they may contain API key material. DEBUG-level logs have the + ... details. + [Arguments] @{args} ${expected_rc}=${0} ${timeout}=120s + ${result}= Run Process ${PYTHON} -m cleveragents @{args} + ... cwd=${WS} + ... env:CLEVERAGENTS_HOME=${SUITE_HOME} + ... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true + ... env:NO_COLOR=1 + ... timeout=${timeout} + ... on_timeout=kill + Log STDOUT: ${result.stdout} level=DEBUG + Log STDERR: ${result.stderr} level=DEBUG + Should Be Equal As Integers ${result.rc} ${expected_rc} + ... CLI failed (rc=${result.rc}). Check DEBUG-level log entries above for stdout/stderr. + RETURN ${result} + +Extract JSON From Stdout + [Documentation] Extract the first JSON object from stdout that may contain + ... leading non-JSON text (e.g. structlog debug messages). + ... Uses ``raw_decode`` with ``strict=False`` to tolerate + ... literal control characters that Rich's ``console.print`` + ... may inject when it wraps long lines inside JSON string + ... values. Wrapped in ``TRY/EXCEPT`` for clear failure messages. + ... + ... **Note:** This picks the *first* JSON object (by scanning for + ... the first ``{``). Non-JSON preamble (e.g. structlog lines) + ... is mitigated by ``NO_COLOR=1`` (disables Rich formatting) and + ... structlog routing to stderr, so stdout typically starts with + ... the JSON payload. See ``Safe Parse Json Field`` in + ... ``common_e2e.resource`` for a last-line fallback strategy + ... that scans in reverse. + [Arguments] ${text} + TRY + ${start}= Evaluate $text.index('{') + ${json_obj}= Evaluate json.JSONDecoder(strict=False).raw_decode($text, $start)[0] modules=json + EXCEPT AS ${err} + Fail Failed to extract JSON object from stdout: ${err} + END + RETURN ${json_obj} + +Link Resource To Project + [Documentation] Link the suite-level workspace resource to a project. + ... + ... Requires the calling suite to set ``${WS_RESOURCE}`` as a + ... suite variable before calling this keyword. + [Arguments] ${project_name} + ${result}= Run CLI project link-resource ${project_name} ${WS_RESOURCE} + Should Be Equal As Integers ${result.rc} 0 + ... msg=project link-resource failed (rc=${result.rc}). Check DEBUG logs above. + +Create Synthetic Codebase + [Documentation] Populate a directory with small Python files and one + ... deliberately large file for budget-enforcement testing. + ... + ... ``${project_label}`` is embedded in docstrings and print + ... statements to distinguish output from different suites. + [Arguments] ${base_dir} ${project_label}=E2E test project + ${main_py}= Catenate SEPARATOR=\n + ... """Main entry point for ${project_label}.""" + ... ${EMPTY} + ... def main() -> None: + ... ${SPACE}${SPACE}${SPACE}${SPACE}"""Run the application.""" + ... ${SPACE}${SPACE}${SPACE}${SPACE}print("Hello from ${project_label}") + ... ${EMPTY} + ... if __name__ == "__main__": + ... ${SPACE}${SPACE}${SPACE}${SPACE}main() + Create File ${base_dir}${/}main.py ${main_py} + ${utils_py}= Catenate SEPARATOR=\n + ... """Utility helpers.""" + ... ${EMPTY} + ... def add(a: int, b: int) -> int: + ... ${SPACE}${SPACE}${SPACE}${SPACE}return a + b + ... ${EMPTY} + ... def multiply(a: int, b: int) -> int: + ... ${SPACE}${SPACE}${SPACE}${SPACE}return a * b + Create File ${base_dir}${/}utils.py ${utils_py} + Create File ${base_dir}${/}config.py TIMEOUT = 30\nMAX_RETRIES = 3\nDEBUG = False + # Large file (>1 KiB) for budget testing + ${large_content}= Evaluate "# auto-generated large file\\n" + ("x = 1\\n" * 250) + Create File ${base_dir}${/}large_file.py ${large_content} + Create Temp Git Repo [Documentation] Create a temporary git repository for E2E testing. ... diff --git a/robot/e2e/m5_acceptance.robot b/robot/e2e/m5_acceptance.robot index 01f62176c..14f541311 100644 --- a/robot/e2e/m5_acceptance.robot +++ b/robot/e2e/m5_acceptance.robot @@ -47,7 +47,7 @@ M5 Acceptance Suite Setup Should Be Equal As Integers ${result.rc} 0 ... Workspace init failed (rc=${result.rc}). Check DEBUG-level log entries above. # Create synthetic source files for context testing - Create Synthetic Codebase ${ws} + Create Synthetic Codebase ${ws} M5 test # Initialize a git repository — check every return code ${git_init}= Run Process git init cwd=${ws} timeout=60s on_timeout=kill Should Be Equal As Integers ${git_init.rc} 0 msg=git init failed (rc=${git_init.rc}). Check DEBUG logs above. @@ -78,53 +78,6 @@ M5 Acceptance Suite Teardown [Documentation] Delegate to the common E2E teardown. E2E Suite Teardown -Create Synthetic Codebase - [Documentation] Populate the workspace with small Python files and one - ... deliberately large file for budget-enforcement testing. - [Arguments] ${base_dir} - ${main_py}= Catenate SEPARATOR=\n - ... """Main entry point for M5 E2E test project.""" - ... ${EMPTY} - ... def main() -> None: - ... ${SPACE}${SPACE}${SPACE}${SPACE}"""Run the application.""" - ... ${SPACE}${SPACE}${SPACE}${SPACE}print("Hello from M5 test") - ... ${EMPTY} - ... if __name__ == "__main__": - ... ${SPACE}${SPACE}${SPACE}${SPACE}main() - Create File ${base_dir}${/}main.py ${main_py} - ${utils_py}= Catenate SEPARATOR=\n - ... """Utility helpers.""" - ... ${EMPTY} - ... def add(a: int, b: int) -> int: - ... ${SPACE}${SPACE}${SPACE}${SPACE}return a + b - ... ${EMPTY} - ... def multiply(a: int, b: int) -> int: - ... ${SPACE}${SPACE}${SPACE}${SPACE}return a * b - Create File ${base_dir}${/}utils.py ${utils_py} - Create File ${base_dir}${/}config.py TIMEOUT = 30\nMAX_RETRIES = 3\nDEBUG = False - # Large file (>1 KiB) for budget testing - ${large_content}= Evaluate "# auto-generated large file\\n" + ("x = 1\\n" * 250) - Create File ${base_dir}${/}large_file.py ${large_content} - -Run CLI - [Documentation] Run ``python -m cleveragents`` inside the workspace directory - ... with the correct environment variables. - [Arguments] @{args} ${expected_rc}=${0} ${timeout}=120s - ${result}= Run Process ${PYTHON} -m cleveragents @{args} - ... cwd=${WS} - ... env:CLEVERAGENTS_HOME=${SUITE_HOME} - ... env:CLEVERAGENTS_AUTO_APPLY_MIGRATIONS=true - ... env:NO_COLOR=1 - ... timeout=${timeout} - ... on_timeout=kill - Log STDOUT: ${result.stdout} level=DEBUG - Log STDERR: ${result.stderr} level=DEBUG - # Do not embed raw stdout/stderr in assertion failure messages — - # they may contain API key material. DEBUG-level logs above have the details. - Should Be Equal As Integers ${result.rc} ${expected_rc} - ... CLI failed (rc=${result.rc}). Check DEBUG-level log entries above for stdout/stderr. - RETURN ${result} - Combined Output [Documentation] Return the concatenation of stdout and stderr (lowercased). [Arguments] ${result} @@ -139,29 +92,6 @@ Skip If No OpenAI Key ${is_missing}= Evaluate len(os.environ.get('OPENAI_API_KEY', '')) == 0 modules=os Skip If ${is_missing} OPENAI_API_KEY not set — required for openai/gpt-4o-mini tests. -Link Resource To Project - [Documentation] Link the suite-level workspace resource to a project. - [Arguments] ${project_name} - ${result}= Run CLI project link-resource ${project_name} ${WS_RESOURCE} - Should Be Equal As Integers ${result.rc} 0 - ... msg=project link-resource failed (rc=${result.rc}). Check DEBUG logs above. - -Extract JSON From Stdout - [Documentation] Extract the first JSON object from stdout that may contain - ... leading non-JSON text (e.g. structlog debug messages). - ... Uses ``raw_decode`` with ``strict=False`` to tolerate - ... literal control characters that Rich's ``console.print`` - ... may inject when it wraps long lines inside JSON string - ... values. Wrapped in ``TRY/EXCEPT`` for clear failure messages. - [Arguments] ${text} - TRY - ${start}= Evaluate $text.index('{') - ${json_obj}= Evaluate json.JSONDecoder(strict=False).raw_decode($text, $start)[0] modules=json - EXCEPT AS ${err} - Fail Failed to extract JSON object from stdout: ${err} - END - RETURN ${json_obj} - Plan Test Setup [Documentation] Setup for plan execution tests — skip if no API key ... and verify prerequisite variables exist. diff --git a/robot/e2e/m6_acceptance.robot b/robot/e2e/m6_acceptance.robot index 2586f3947..dfc928102 100644 --- a/robot/e2e/m6_acceptance.robot +++ b/robot/e2e/m6_acceptance.robot @@ -26,11 +26,11 @@ M6 Suite Setup Set Suite Variable ${RUN_SUFFIX} ${suffix} # Register the local/code-review action needed by plan lifecycle tests. # Pick an actor that matches the available API key. - ${has_anthropic}= Evaluate bool(__import__('os').environ.get('ANTHROPIC_API_KEY', '')) - IF ${has_anthropic} - ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 - ELSE + ${has_openai}= Evaluate bool(__import__('os').environ.get('OPENAI_API_KEY', '')) + IF ${has_openai} ${actor}= Set Variable openai/gpt-4o + ELSE + ${actor}= Set Variable anthropic/claude-sonnet-4-20250514 END ${action_yaml}= Catenate SEPARATOR=\n ... name: local/code-review diff --git a/robot/e2e/tdd_acms_behavioral_validation.robot b/robot/e2e/tdd_acms_behavioral_validation.robot new file mode 100644 index 000000000..34b547132 --- /dev/null +++ b/robot/e2e/tdd_acms_behavioral_validation.robot @@ -0,0 +1,271 @@ +*** Settings *** +Documentation TDD Bug #1028 — ACMS indexing pipeline not wired into CLI. +... +... Behavioral E2E tests proving that ``ContextTierService`` starts +... empty on every CLI invocation because the ACMS indexing pipeline +... is not wired into the CLI entry points. The ``project context +... simulate`` and ``project context inspect`` commands operate on +... zero data even when run against a project directory containing +... files. +... +... All tests are tagged ``tdd_expected_fail`` so the +... ``tdd_expected_fail_listener`` inverts results — a failing +... assertion (proving the bug exists) passes CI, while a passing +... assertion (bug appears fixed) fails CI until the tag is +... removed by the bug-fix developer. +... +... See CONTRIBUTING.md > Bug Fix Workflow for the full TDD +... bug-capture lifecycle. +... +... **Known limitation — result inversion scope:** +... The ``tdd_expected_fail`` listener inverts the *entire* test +... outcome, not just specific assertions. If a test fails for an +... unrelated reason (e.g. CLI crash, infrastructure error, or suite +... setup failure), the inversion still converts that failure to +... PASS — producing a false-positive "bug confirmed" result. +... This is accepted because the corresponding non-inverted +... acceptance tests in ``m5_acceptance.robot`` cover the same CLI +... plumbing structurally, so infrastructure regressions surface +... there even if masked here. The bug-fix developer should run +... these tests *without* the ``tdd_expected_fail`` tag to verify +... genuine assertion results after wiring the indexing pipeline. +Resource common_e2e.resource +Library OperatingSystem +Library String +Library Collections +Library Process +Suite Setup ACMS Behavioral Suite Setup +Suite Teardown ACMS Behavioral Suite Teardown + +*** Variables *** +${PROJECT_SIMULATE} local/tdd-1028-simulate +${PROJECT_INSPECT} local/tdd-1028-inspect +${PROJECT_BUDGET} local/tdd-1028-budget +${PROJECT_SCALE} local/tdd-1028-scale + +*** Keywords *** +ACMS Behavioral Suite Setup + [Documentation] Create an isolated workspace with ``agents init``, build a + ... synthetic codebase, initialise a git repo, register it as a + ... resource, and prepare for behavioral ACMS tests. + E2E Suite Setup + # Create workspace directory inside the suite home + ${ws}= Set Variable ${SUITE_HOME}${/}workspace + Create Directory ${ws} + Set Suite Variable ${WS} ${ws} + # Initialize the CleverAgents workspace + ${result}= Run CLI init m5-tdd-workspace + Log Init stdout: ${result.stdout} level=DEBUG + Log Init stderr: ${result.stderr} level=DEBUG + Should Be Equal As Integers ${result.rc} 0 + ... Workspace init failed (rc=${result.rc}). Check DEBUG-level log entries above. + # Create synthetic source files for context testing + Create Synthetic Codebase ${ws} TDD 1028 test + # Initialize a git repository — check every return code + ${git_init}= Run Process git init cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_init.rc} 0 msg=git init failed (rc=${git_init.rc}). Check DEBUG logs above. + ${git_cfg_name}= Run Process git config user.name E2E Test cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_cfg_name.rc} 0 msg=git config user.name failed (rc=${git_cfg_name.rc}). Check DEBUG logs above. + ${git_cfg_email}= Run Process git config user.email e2e@test.local cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_cfg_email.rc} 0 msg=git config user.email failed (rc=${git_cfg_email.rc}). Check DEBUG logs above. + ${git_add}= Run Process git add . cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_add.rc} 0 msg=git add failed (rc=${git_add.rc}). Check DEBUG logs above. + ${git_commit}= Run Process git commit -m Initial commit cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${git_commit.rc} 0 msg=git commit failed (rc=${git_commit.rc}). Check DEBUG logs above. + # Detect the default branch created by git init + ${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${ws} timeout=60s on_timeout=kill + Should Be Equal As Integers ${branch_result.rc} 0 msg=git rev-parse failed (rc=${branch_result.rc}). Check DEBUG logs above. + ${branch}= Strip String ${branch_result.stdout} + Set Suite Variable ${WS_BRANCH} ${branch} + # Register the workspace as a git-checkout resource + ${res_name}= Set Variable local/tdd-1028-ws-resource + Set Suite Variable ${WS_RESOURCE} ${res_name} + ${r_add}= Run CLI resource add git-checkout ${res_name} --path ${ws} --branch ${branch} + Should Be Equal As Integers ${r_add.rc} 0 msg=resource add failed (rc=${r_add.rc}). Check DEBUG logs above. + Set Suite Variable ${SUITE_SETUP_COMPLETE} ${TRUE} + +ACMS Behavioral Suite Teardown + [Documentation] Delegate to the common E2E teardown. + E2E Suite Teardown + +*** Test Cases *** +# ----------------------------------------------------------------------- +# TDD Bug #1028 — Behavioral ACMS validation +# +# NOTE: Each test guards against incomplete suite setup with +# [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} +# If suite setup fails, this guard raises an error which the +# tdd_expected_fail listener inverts to PASS — silently passing +# the test without executing any assertions. This is a known +# limitation of the tdd_expected_fail pattern; see the suite +# documentation above. The non-inverted m5_acceptance.robot +# tests cover the same CLI plumbing and will surface setup +# failures independently. +# ----------------------------------------------------------------------- + +Context Simulate Returns Non-Empty Tier Data + [Documentation] Run ``project context simulate`` against a project with + ... files and assert that the output contains actual indexed + ... fragments (not empty tiers). + ... + ... **Expected bug behavior:** ``total_tokens`` is 0 and + ... ``fragment_count`` is 0 because the ACMS indexing pipeline + ... is not wired into the CLI — ``ContextTierService`` starts + ... empty on every invocation. + ... + ... Tagged ``tdd_expected_fail`` because the assertion + ... ``fragment_count > 0`` will fail, proving the bug. + [Tags] tdd_expected_fail tdd_bug tdd_bug_1028 E2E + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + ... msg=Prerequisite not met: suite setup did not complete + # Create project and configure context policy + Run CLI project create ${PROJECT_SIMULATE} + Link Resource To Project ${PROJECT_SIMULATE} + Run CLI + ... project context set ${PROJECT_SIMULATE} + ... --view default + ... --include-path **/*.py + # Run simulate and check for actual indexed data + ${result}= Run CLI + ... project context simulate ${PROJECT_SIMULATE} + ... --format json + ${sim_json}= Extract JSON From Stdout ${result.stdout} + # Behavioral assertion: the project has Python files, so simulate + # must produce non-zero fragment data after indexing. + ${fragment_count}= Evaluate int($sim_json.get('fragment_count', 0)) + Should Be True ${fragment_count} > 0 + ... msg=Bug #1028: fragment_count is ${fragment_count} (expected > 0). ACMS indexing pipeline is not wired into CLI — ContextTierService starts empty. + ${total_tokens}= Evaluate int($sim_json.get('total_tokens', 0)) + Should Be True ${total_tokens} > 0 + ... msg=Bug #1028: total_tokens is ${total_tokens} (expected > 0). No fragments were indexed from project files. + +Context Inspect Shows Indexed Resources + [Documentation] Run ``project context inspect`` against a project with + ... files and assert that the indexed resource count is > 0. + ... + ... **Expected bug behavior:** ``tier_metrics`` counters are + ... all zero because no indexing occurs — the ACMS pipeline + ... is disconnected from the CLI. + ... + ... Tagged ``tdd_expected_fail`` because the assertion on + ... non-zero tier counts will fail, proving the bug. + [Tags] tdd_expected_fail tdd_bug tdd_bug_1028 E2E + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + ... msg=Prerequisite not met: suite setup did not complete + # Create project and configure context policy + Run CLI project create ${PROJECT_INSPECT} + Link Resource To Project ${PROJECT_INSPECT} + Run CLI + ... project context set ${PROJECT_INSPECT} + ... --view default + ... --include-path **/*.py + # Run inspect and check for indexed resources + ${result}= Run CLI + ... project context inspect ${PROJECT_INSPECT} + ... --format json + ${inspect_json}= Extract JSON From Stdout ${result.stdout} + # Behavioral assertion: at least one tier should have fragments + ${metrics}= Evaluate $inspect_json.get('tier_metrics', {}) + ${hot_count}= Evaluate int($metrics.get('hot_count', 0)) + ${warm_count}= Evaluate int($metrics.get('warm_count', 0)) + ${cold_count}= Evaluate int($metrics.get('cold_count', 0)) + ${total_indexed}= Evaluate ${hot_count} + ${warm_count} + ${cold_count} + Should Be True ${total_indexed} > 0 + ... msg=Bug #1028: total indexed fragments is ${total_indexed} (expected > 0). tier_metrics: hot=${hot_count}, warm=${warm_count}, cold=${cold_count}. ACMS indexing pipeline is not wired into CLI. + +Budget Enforcement Excludes Oversized Files + [Documentation] Configure ``max_file_size`` policy, add a file exceeding + ... that limit, run simulate, and assert that the oversized + ... file is excluded while smaller files are indexed. + ... + ... **Expected bug behavior:** ``fragment_count`` is 0 because + ... the indexing pipeline does not run at all — regardless of + ... ``max_file_size`` configuration, no files are scanned. + ... + ... Tagged ``tdd_expected_fail`` because the assertion on + ... ``fragment_count > 0`` will fail, proving the bug. + [Tags] tdd_expected_fail tdd_bug tdd_bug_1028 E2E + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + ... msg=Prerequisite not met: suite setup did not complete + # Create project with tight max_file_size (1024 bytes) + Run CLI project create ${PROJECT_BUDGET} + Link Resource To Project ${PROJECT_BUDGET} + Run CLI + ... project context set ${PROJECT_BUDGET} + ... --view default + ... --include-path **/*.py + ... --max-file-size 1024 + ... --max-total-size 8192 + # Simulate should index small files but exclude large_file.py (>1KiB) + ${result}= Run CLI + ... project context simulate ${PROJECT_BUDGET} + ... --format json + ${sim_json}= Extract JSON From Stdout ${result.stdout} + # Behavioral assertion: at least the small files (main.py, utils.py, + # config.py) should be indexed — large_file.py should be excluded by + # max_file_size policy. If fragment_count > 0, some files were indexed + # (and budget enforcement partially works). + # TODO(bugfix/m5-acms-cli-indexing-pipeline-wiring): After the indexing + # pipeline is wired, add a second assertion verifying that large_file.py + # is absent from the fragment list (i.e. budget enforcement actually + # excludes oversized files, not just that *some* files are indexed). + ${fragment_count}= Evaluate int($sim_json.get('fragment_count', 0)) + Should Be True ${fragment_count} > 0 + ... msg=Bug #1028: fragment_count is ${fragment_count} (expected > 0). Budget enforcement cannot exclude oversized files because the indexing pipeline does not run at all. + +Large Project Indexes Without Timeout + [Documentation] Create a synthetic 10,000+ file project, run + ... ``project context simulate``, and assert completion + ... within a reasonable timeout with non-empty results. + ... + ... **Expected bug behavior:** The simulate command completes + ... but returns zero fragments because the indexing pipeline + ... is not wired — the 10K files are never scanned. + ... + ... Tagged ``tdd_expected_fail`` because the assertion on + ... ``fragment_count > 0`` will fail, proving the bug. + [Tags] tdd_expected_fail tdd_bug tdd_bug_1028 E2E + [Setup] Variable Should Exist ${SUITE_SETUP_COMPLETE} + ... msg=Prerequisite not met: suite setup did not complete + # Generate 10,000 tiny .py files in a subdirectory + ${scale_dir}= Set Variable ${WS}${/}scale_src + Create Directory ${scale_dir} + ${script}= Catenate SEPARATOR=\n + ... import os, sys + ... d = sys.argv[1] + ... for i in range(10000): + ... ${SPACE}${SPACE}${SPACE}${SPACE}with open(os.path.join(d, f"mod_{i:05d}.py"), "w") as f: + ... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}f.write(f"# module {i}\\ndef fn_{i}(): return {i}\\n") + ${gen_script}= Set Variable ${WS}${/}_gen_10k.py + Create File ${gen_script} ${script} + ${gen}= Run Process ${PYTHON} ${gen_script} ${scale_dir} + ... timeout=120s on_timeout=kill + Should Be Equal As Integers ${gen.rc} 0 msg=10K file generation failed + # NOTE: The 10K generated files are written to the filesystem but NOT + # committed to the workspace git repo. The resource is registered as + # git-checkout type. The bug-fix developer MUST evaluate whether the + # fix routes indexing through the git sandbox (which only exposes + # git-tracked content) or the filesystem. If the fix uses git-tracked + # content, uncomment the following lines to commit the generated files: + # Run Process git add scale_src/ cwd=${WS} + # Run Process git commit -m Add 10K test files cwd=${WS} + # The m5_acceptance.robot structural test has the same pattern — + # neither suite commits generated files. + Remove File ${gen_script} + # Create project with include path covering the 10K files + Run CLI project create ${PROJECT_SCALE} + Link Resource To Project ${PROJECT_SCALE} + Run CLI + ... project context set ${PROJECT_SCALE} + ... --view default + ... --include-path scale_src/**/*.py + # Simulate must complete within 600s (10 minutes) and return data + ${result}= Run CLI + ... project context simulate ${PROJECT_SCALE} + ... --format json timeout=600s + ${sim_json}= Extract JSON From Stdout ${result.stdout} + # Behavioral assertion: with 10,000+ files, fragment_count must be + # non-zero if the indexing pipeline is operational. + ${fragment_count}= Evaluate int($sim_json.get('fragment_count', 0)) + Should Be True ${fragment_count} > 0 + ... msg=Bug #1028: fragment_count is ${fragment_count} (expected > 0). 10,000+ files were generated but none were indexed. ACMS indexing pipeline is not wired into CLI.