|
|
|
@@ -0,0 +1,166 @@
|
|
|
|
|
"""Step definitions for TDD session tell --stream redaction tests.
|
|
|
|
|
|
|
|
|
|
These tests verify that ``session tell --stream`` routes output through the
|
|
|
|
|
Rich console (and therefore through the redaction layer) rather than writing
|
|
|
|
|
directly to ``sys.stdout``.
|
|
|
|
|
|
|
|
|
|
Issue #10458 (TDD): Write failing test capturing the bug.
|
|
|
|
|
Issue #10460 (Fix): Replace sys.stdout.write loop with console.print.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
|
|
|
|
|
|
from behave import given, then, when
|
|
|
|
|
from typer.testing import CliRunner
|
|
|
|
|
|
|
|
|
|
from cleveragents.cli.commands.session import app as session_app
|
|
|
|
|
from cleveragents.domain.models.core.session import SessionService
|
|
|
|
|
|
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
|
|
|
|
_ULID = "01KJ1N8YDN05N29P5RZV4SPDVZ"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _mock_service() -> MagicMock:
|
|
|
|
|
"""Create a MagicMock that passes isinstance checks for SessionService."""
|
|
|
|
|
svc = MagicMock(spec=SessionService)
|
|
|
|
|
svc.append_message.return_value = None
|
|
|
|
|
return svc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _patch_service(context: object, svc: MagicMock) -> None:
|
|
|
|
|
"""Patch session._service and register cleanup."""
|
|
|
|
|
patcher = patch("cleveragents.cli.commands.session._service", svc)
|
|
|
|
|
patcher.start()
|
|
|
|
|
context._cleanup_handlers.append(patcher.stop) # type: ignore[attr-defined]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Background
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@given("a session tell stream redaction mock service")
|
|
|
|
|
def step_setup_mock_service(context: object) -> None:
|
|
|
|
|
svc = _mock_service()
|
|
|
|
|
_patch_service(context, svc)
|
|
|
|
|
context.mock_svc = svc # type: ignore[attr-defined]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# When steps
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@when("I invoke session tell with --stream and a sensitive prompt")
|
|
|
|
|
def step_invoke_tell_stream_sensitive(context: object) -> None:
|
|
|
|
|
"""Invoke tell --stream with a prompt that contains a sensitive-looking value."""
|
|
|
|
|
sensitive_prompt = "my-api-key=sk-secret1234567890abcdef"
|
|
|
|
|
context.result = runner.invoke( # type: ignore[attr-defined]
|
|
|
|
|
session_app,
|
|
|
|
|
["tell", "--session", _ULID, "--stream", sensitive_prompt],
|
|
|
|
|
)
|
|
|
|
|
context.prompt_used = sensitive_prompt # type: ignore[attr-defined]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@when("I invoke session tell with --stream and a plain prompt")
|
|
|
|
|
def step_invoke_tell_stream_plain(context: object) -> None:
|
|
|
|
|
"""Invoke tell --stream with a plain prompt."""
|
|
|
|
|
context.result = runner.invoke( # type: ignore[attr-defined]
|
|
|
|
|
session_app,
|
|
|
|
|
["tell", "--session", _ULID, "--stream", "Hello world"],
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Then steps
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@then("the tell command exits with code 0")
|
|
|
|
|
def step_exit_code_0(context: object) -> None:
|
|
|
|
|
result = context.result # type: ignore[attr-defined]
|
|
|
|
|
assert result.exit_code == 0, (
|
|
|
|
|
f"Expected exit code 0, got {result.exit_code}.\nOutput:\n{result.output}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@then("sys.stdout.write was not called directly during streaming")
|
|
|
|
|
def step_stdout_write_not_called(context: object) -> None:
|
|
|
|
|
"""Verify that the streaming path does NOT call sys.stdout.write directly.
|
|
|
|
|
|
|
|
|
|
After the fix, the streaming path uses console.print() instead of
|
|
|
|
|
sys.stdout.write(). We verify this by checking that the output was
|
|
|
|
|
captured by the CliRunner (which captures Rich console output) rather
|
|
|
|
|
than going through raw sys.stdout.write calls.
|
|
|
|
|
|
|
|
|
|
The CliRunner from typer.testing captures output written through the
|
|
|
|
|
Rich console. If sys.stdout.write were called directly with the
|
|
|
|
|
character-by-character loop, the output would still appear in
|
|
|
|
|
result.output (since CliRunner redirects stdout), but the key
|
|
|
|
|
behavioral difference is that console.print applies markup/redaction.
|
|
|
|
|
|
|
|
|
|
We verify the fix by re-running with a patched sys.stdout and confirming
|
|
|
|
|
that sys.stdout.write is NOT called with individual characters.
|
|
|
|
|
"""
|
|
|
|
|
import sys
|
|
|
|
|
from unittest.mock import patch as mock_patch
|
|
|
|
|
|
|
|
|
|
write_calls: list[str] = []
|
|
|
|
|
|
|
|
|
|
original_write = sys.stdout.write
|
|
|
|
|
|
|
|
|
|
def tracking_write(s: str) -> int:
|
|
|
|
|
write_calls.append(s)
|
|
|
|
|
return original_write(s)
|
|
|
|
|
|
|
|
|
|
svc = _mock_service()
|
|
|
|
|
sensitive_prompt = "my-api-key=sk-secret1234567890abcdef"
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
mock_patch("cleveragents.cli.commands.session._service", svc),
|
|
|
|
|
mock_patch("sys.stdout.write", side_effect=tracking_write),
|
|
|
|
|
):
|
|
|
|
|
runner.invoke(
|
|
|
|
|
session_app,
|
|
|
|
|
["tell", "--session", _ULID, "--stream", sensitive_prompt],
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# After the fix: sys.stdout.write should NOT be called with individual
|
|
|
|
|
# characters (single-char strings). The old buggy code called
|
|
|
|
|
# sys.stdout.write(char) for each character in assistant_content.
|
|
|
|
|
single_char_writes = [c for c in write_calls if len(c) == 1]
|
|
|
|
|
assert len(single_char_writes) == 0, (
|
|
|
|
|
f"sys.stdout.write was called with {len(single_char_writes)} individual "
|
|
|
|
|
f"characters, indicating the streaming path still uses the direct "
|
|
|
|
|
f"sys.stdout.write loop instead of console.print. "
|
|
|
|
|
f"First few chars: {single_char_writes[:10]}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@then('the tell output contains "{text}"')
|
|
|
|
|
def step_output_contains(context: object, text: str) -> None:
|
|
|
|
|
result = context.result # type: ignore[attr-defined]
|
|
|
|
|
assert text in result.output, f"Expected '{text}' in output.\nGot:\n{result.output}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@then("the streaming output was written through the Rich console")
|
|
|
|
|
def step_output_via_console(context: object) -> None:
|
|
|
|
|
"""Verify that the streaming output appears in the CliRunner-captured output.
|
|
|
|
|
|
|
|
|
|
The CliRunner captures output written through the Rich console (which
|
|
|
|
|
writes to stdout). After the fix, console.print(assistant_content, end="")
|
|
|
|
|
is used, so the full assistant content should appear in result.output
|
|
|
|
|
as a single write rather than character-by-character.
|
|
|
|
|
|
|
|
|
|
We verify that the output contains the expected assistant response and
|
|
|
|
|
that it was NOT split into individual characters in the captured output.
|
|
|
|
|
"""
|
|
|
|
|
result = context.result # type: ignore[attr-defined]
|
|
|
|
|
# The assistant content for "Hello world" is "Acknowledged: Hello world"
|
|
|
|
|
expected = "Acknowledged: Hello world"
|
|
|
|
|
assert expected in result.output, (
|
|
|
|
|
f"Expected '{expected}' in streaming output.\nGot:\n{result.output}"
|
|
|
|
|
)
|