cleveragents-core/features/steps/llm_delimiter_regression_steps.py

"""Step definitions for llm_delimiter_regression.feature (#10878).

Re-proves that the CLEVERAGENTS_FILE_START/END sentinel scheme fixes
the triple-backtick delimiter collision bug from #10878.

Two in-line "old" parsers are provided to demonstrate the original bug:
  * _old_backtick_nongreedy  -- non-greedy ```.``` pattern; truncates content
    at the first triple-backtick *inside* a file body but still finds all
    FILE blocks (each with truncated content).
  * _old_backtick_greedy     -- greedy ```.``` pattern; swallows subsequent
    FILE blocks inside the first match's "content", returning far fewer
    entries than expected.

The current ``LLMExecuteActor._parse_file_blocks`` uses
``<<<<<<< CLEVERAGENTS_FILE_START >>>>>>>`` / ``<<<<<<< CLEVERAGENTS_FILE_END >>>>>>>``
(and the newer ``<CAFS>``/``</CAFE>``) markers, which never collide with
triple-backtick Markdown fences.

Forgejo: #10878
"""

from __future__ import annotations

import re
import subprocess

from behave import given, step, then, when  # type: ignore[import-untyped]

from cleveragents.application.services.llm_actors import LLMExecuteActor


# ---------------------------------------------------------------------------
# Old (buggy) backtick-based parsers used to reproduce the pre-fix behaviour
# ---------------------------------------------------------------------------

_LEGACY_START = "<<<<<<< CLEVERAGENTS_FILE_START >>>>>>>"
_LEGACY_END = "<<<<<<< CLEVERAGENTS_FILE_END >>>>>>>"


def _old_backtick_nongreedy(llm_output: str) -> list[str]:
    """Non-greedy old parser.

    Finds the correct number of FILE blocks but truncates their content at
    the first triple-backtick that appears inside a file body.
    """
    pattern = re.compile(
        r"FILE:\s*(.+?)\s*\n```\n(.*?)\n```",
        re.DOTALL,
    )
    return [m.group(1).strip() for m in pattern.finditer(llm_output)]


def _old_backtick_greedy(llm_output: str) -> list[str]:
    """Greedy old parser.

    The greedy ``(.*)`` causes the *first* match to consume all remaining
    content up to the very last triple-backtick line in the document,
    swallowing every subsequent FILE block inside the captured "content".
    Only one (or very few) entries are returned for an input that contains
    many FILE blocks.
    """
    pattern = re.compile(
        r"FILE:\s*(.+?)\s*\n```\n(.*)\n```",
        re.DOTALL,
    )
    return [m.group(1).strip() for m in pattern.finditer(llm_output)]


# ---------------------------------------------------------------------------
# Helper builders
# ---------------------------------------------------------------------------


def _make_legacy_block(path: str, content: str) -> str:
    """FILE block using the CLEVERAGENTS_FILE_START/END markers."""
    return f"FILE: {path}\n{_LEGACY_START}\n{content}\n{_LEGACY_END}\n\n"


def _make_old_backtick_block(path: str, content: str) -> str:
    """FILE block using the old triple-backtick delimiters."""
    return f"FILE: {path}\n```\n{content}\n```\n\n"


def _fence_content(label: str) -> str:
    """File body that embeds a triple-backtick Python code fence."""
    return (
        f"# {label}\n"
        "```python\n"
        f"def {label.replace(' ', '_')}(): pass\n"
        "```\n"
        f"# end of {label}\n"
    )


# ---------------------------------------------------------------------------
# Given — context setup
# ---------------------------------------------------------------------------


@given("a string containing ````")
def step_context_with_backtick_string(context):  # type: ignore[no-untyped-def]
    context.backtick_context = "````"


@given("a string ````")
def step_short_backtick_ctx(context):  # type: ignore[no-untyped-def]
    context.backtick_context = "````"


@given('a mock plan_id "01HQXXXXX"')
def step_plan_id_xxxxx(context):  # type: ignore[no-untyped-def]
    context.delimiter_plan_id = "01HQXXXXX"


@given('a mock plan_id "01HYYYYYY"')
def step_plan_id_yyyyyy(context):  # type: ignore[no-untyped-def]
    context.delimiter_plan_id = "01HYYYYYY"


# --- Input-creation steps ---


@step("I create file with LLM output containing code fences in body")
def step_create_old_2files_with_fences(context):  # type: ignore[no-untyped-def]
    """Two FILE blocks in old backtick format, each with an embedded code fence.

    The non-greedy old parser will find both files but truncate their content
    at the first triple-backtick in each body (the opening fence).
    """
    context.delim_llm_output = _make_old_backtick_block(
        "src/main.py", _fence_content("main")
    ) + _make_old_backtick_block("src/util.py", _fence_content("util"))


@step(
    "I create file with LLM output using unique delimiters and code fences embedded in body"
)
def step_create_new_6files_with_fences(context):  # type: ignore[no-untyped-def]
    """Six FILE blocks using CLEVERAGENTS markers with embedded code fences."""
    context.delim_llm_output = "".join(
        _make_legacy_block(f"src/mod_{i}.py", _fence_content(f"mod_{i}"))
        for i in range(1, 7)
    )


@step("I create LLM output where file content contains embedded markdown code blocks")
def step_create_old_4files_greedy(context):  # type: ignore[no-untyped-def]
    """Four FILE blocks in old backtick format, each with embedded code fences.

    The *greedy* old parser will match only the first FILE block and
    consume the remaining three as part of the first block's "content".
    """
    context.delim_llm_output = (
        _make_old_backtick_block("src/a.py", _fence_content("a"))
        + _make_old_backtick_block("src/b.py", _fence_content("b"))
        + _make_old_backtick_block("src/c.py", _fence_content("c"))
        + _make_old_backtick_block("src/d.py", _fence_content("d"))
    )


@step(
    "I create LLM output using unique sentinels with embedded ```python code blocks in body"
)
def step_create_new_4files_with_fences(context):  # type: ignore[no-untyped-def]
    """Four FILE blocks using CLEVERAGENTS markers with embedded code fences."""
    context.delim_llm_output = (
        _make_legacy_block("src/a.py", _fence_content("a"))
        + _make_legacy_block("src/b.py", _fence_content("b"))
        + _make_legacy_block("src/c.py", _fence_content("c"))
        + _make_legacy_block("src/d.py", _fence_content("d"))
    )


@step(
    "I create full regression file block output that includes report section plus"
    " multiple files each with inline ```python examples"
)
def step_create_full_regression_output(context):  # type: ignore[no-untyped-def]
    """Architecture-report header followed by five FILE blocks with CLEVERAGENTS markers."""
    prose = "## Architecture Report\n\nThis section describes key modules.\n\n"
    files = [
        _make_legacy_block(f"src/mod_{i}.py", _fence_content(f"mod_{i}"))
        for i in range(1, 6)
    ]
    context.delim_llm_output = prose + "".join(files)
    context.delim_expected_count = 5


@step(
    "LLM output containing two FILE blocks each with embedded"
    " ```python sections in the content"
)
def step_create_new_2files_with_python_fences(context):  # type: ignore[no-untyped-def]
    """Two FILE blocks with CLEVERAGENTS markers and embedded Python code fences."""
    context.delim_llm_output = _make_legacy_block(
        "mod_a.py", _fence_content("module_a")
    ) + _make_legacy_block("src/mod_b.py", _fence_content("module_b"))


@step(
    "an llm_output where the file body ends on a line that is exactly ``` after content"
)
def step_create_trailing_backtick_body(context):  # type: ignore[no-untyped-def]
    """Single FILE block whose last body line is exactly three backticks."""
    context.delim_llm_output = (
        f"FILE: module.py\n{_LEGACY_START}\nline one\nline two\n```\n{_LEGACY_END}\n\n"
    )


@step(
    "LLM input with backtick-delimited FILE blocks containing code without"
    " triple-backticks inside body"
)
def step_create_single_simple_block(context):  # type: ignore[no-untyped-def]
    """Single FILE block with CLEVERAGENTS markers and plain content (no code fences).

    This tests backward-compatible parsing of simple single-file blocks.
    """
    context.delim_llm_output = _make_legacy_block(
        "src/simple.py",
        "# Simple module\ndef run(): return 0\n",
    )


# ---------------------------------------------------------------------------
# When — command-like no-ops (side-effect-free shell invocations)
# ---------------------------------------------------------------------------


@when("I run the command \"echo 'backtick-test'\"")
def step_run_echo_backtick_test(context):  # type: ignore[no-untyped-def]
    subprocess.run(["echo", "backtick-test"], capture_output=True, check=False)


@when("I run the command \"echo 'old-parser-test'\"")
def step_run_echo_old_parser_test(context):  # type: ignore[no-untyped-def]
    subprocess.run(["echo", "old-parser-test"], capture_output=True, check=False)


@when("I run the command \"echo 'new-parser-test'\"")
def step_run_echo_new_parser_test(context):  # type: ignore[no-untyped-def]
    subprocess.run(["echo", "new-parser-test"], capture_output=True, check=False)


@when("I run the command \"echo 'full-regression-test'\"")
def step_run_echo_full_regression(context):  # type: ignore[no-untyped-def]
    subprocess.run(["echo", "full-regression-test"], capture_output=True, check=False)


@when("I run the command \"echo 'backtick-sanitise'\"")
def step_run_echo_backtick_sanitise(context):  # type: ignore[no-untyped-def]
    subprocess.run(["echo", "backtick-sanitise"], capture_output=True, check=False)


# --- Parsing steps ---


@when("I parse the LLM output as if it were old backtick-delimited file blocks")
def step_parse_old_nongreedy(context):  # type: ignore[no-untyped-def]
    context.delim_entries = _old_backtick_nongreedy(context.delim_llm_output)


@when(
    "I parse the LLM output with new"
    " CLEVERAGENTS_FILE_START/CLEVERAGENTS_FILE_END delimiters"
)
def step_parse_new_cleveragents(context):  # type: ignore[no-untyped-def]
    plan_id = getattr(context, "delimiter_plan_id", "DELIM_PLAN")
    context.delim_entries, _ = LLMExecuteActor._parse_file_blocks(
        context.delim_llm_output, plan_id
    )


@when("I parse it with the old backtick-style delimiter pattern")
def step_parse_old_greedy(context):  # type: ignore[no-untyped-def]
    context.delim_entries = _old_backtick_greedy(context.delim_llm_output)


@when("I parse it with CLEVERAGENTS_FILE_START / CLEVERAGENTS_FILE_END delimiters")
def step_parse_new_4files(context):  # type: ignore[no-untyped-def]
    plan_id = getattr(context, "delimiter_plan_id", "DELIM_PLAN")
    context.delim_entries, _ = LLMExecuteActor._parse_file_blocks(
        context.delim_llm_output, plan_id
    )


@when("I parse it with new CLEVERAGENTS_FILE_START / CLEVERAGENTS_FILE_END delimiters")
def step_parse_new_full_regression(context):  # type: ignore[no-untyped-def]
    plan_id = getattr(context, "delimiter_plan_id", "DELIM_PLAN")
    context.delim_entries, _ = LLMExecuteActor._parse_file_blocks(
        context.delim_llm_output, plan_id
    )


@when("I call _parse_file_blocks on that LLM output")
def step_call_parse_file_blocks_llm(context):  # type: ignore[no-untyped-def]
    plan_id = getattr(context, "delimiter_plan_id", "DELIM_PLAN")
    context.delim_entries, _ = LLMExecuteActor._parse_file_blocks(
        context.delim_llm_output, plan_id
    )


@when("I call _parse_file_blocks on that output")
def step_call_parse_file_blocks(context):  # type: ignore[no-untyped-def]
    plan_id = getattr(context, "delimiter_plan_id", "DELIM_PLAN")
    context.delim_entries, _ = LLMExecuteActor._parse_file_blocks(
        context.delim_llm_output, plan_id
    )


# ---------------------------------------------------------------------------
# Then — assertions
# ---------------------------------------------------------------------------


@then("the parser should return only 2 entries truncated at first triple-backtick")
def step_assert_2_truncated_entries(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 2, (
        f"Expected 2 entries (truncated) from old backtick parser, got {len(entries)}: "
        f"{entries}"
    )


@then("the parser should return 6 entries not truncated at any triple-backtick")
def step_assert_6_full_entries(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 6, (
        f"Expected 6 complete entries from new CLEVERAGENTS parser, got {len(entries)}"
    )


@then("the parser should return only 1 entry instead of the expected 4")
def step_assert_1_instead_of_4(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 1, (
        f"Expected greedy old parser to return 1 (swallowing other files), "
        f"got {len(entries)}: {entries}"
    )


@then("the parser should return a result count of 4 entries")
def step_assert_4_entries(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 4, (
        f"Expected 4 entries from new CLEVERAGENTS parser, got {len(entries)}"
    )


@then("the parser should return the full entry count of all embedded blocks")
def step_assert_full_entry_count(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    expected = getattr(context, "delim_expected_count", 5)
    assert len(entries) == expected, (
        f"Expected {expected} entries (full regression), got {len(entries)}"
    )


@then("it should return 2 changeset entries not 1")
def step_assert_2_not_1(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 2, (
        f"Expected 2 entries (new parser handles embedded fences, not 1): "
        f"got {len(entries)}"
    )


@then("it should return 1 entry")
def step_assert_1_entry(context):  # type: ignore[no-untyped-def]
    entries = context.delim_entries
    assert len(entries) == 1, f"Expected exactly 1 entry, got {len(entries)}"