Files
cleveragents-core/features/steps/session_cli_steps.py
T
hurui200320 85c579b51f
CI / status-check (push) Blocked by required conditions
CI / benchmark-regression (push) Waiting to run
CI / push-validation (push) Successful in 36s
CI / helm (push) Successful in 45s
CI / build (push) Successful in 58s
CI / lint (push) Successful in 1m9s
CI / quality (push) Successful in 1m18s
CI / typecheck (push) Successful in 1m31s
CI / security (push) Successful in 1m36s
CI / e2e_tests (push) Successful in 3m42s
CI / unit_tests (push) Successful in 4m38s
CI / integration_tests (push) Successful in 4m51s
CI / coverage (push) Has started running
CI / docker (push) Successful in 1m30s
CI / benchmark-publish (push) Has started running
CI / helm (pull_request) Successful in 32s
CI / push-validation (pull_request) Successful in 23s
CI / build (pull_request) Successful in 1m2s
CI / lint (pull_request) Successful in 1m11s
CI / quality (pull_request) Successful in 1m16s
CI / typecheck (pull_request) Successful in 1m24s
CI / security (pull_request) Successful in 1m38s
CI / benchmark-publish (pull_request) Has been skipped
CI / e2e_tests (pull_request) Successful in 4m36s
CI / unit_tests (pull_request) Successful in 4m48s
CI / benchmark-regression (pull_request) Failing after 1m23s
CI / docker (pull_request) Successful in 1m49s
CI / coverage (pull_request) Successful in 10m54s
CI / integration_tests (pull_request) Failing after 3m59s
CI / status-check (pull_request) Failing after 4s
fix(cli): display full session IDs in session list output
Remove the [:8] truncation from session IDs in the Rich table row,
Most Recent summary, and Oldest summary fields of list_sessions()
and _session_list_dict().  Session IDs are 26-character ULIDs and
must be usable directly for copy-paste into session tell, session
show, and other session commands.  The structured output (JSON/YAML)
already used full IDs in the sessions[*].id field, but the summary
panel leaked the truncation into those formats as well.

Added Behave scenarios:
- Rich table displays full 26-character ULIDs (scoped to table region)
- Summary panel shows full ULIDs for unnamed sessions (scoped to panel)
- Summary panel shows session names for named sessions
- Full ULID from list output works with session tell (round-trip,
  uses parsed ULID, not hardcoded constant)

Review Cycle 2 fixes:
- docs/specification.md: Updated YAML output example to full ULIDs
- docs/showcase/*.md: Updated all example output blocks to full ULIDs
- docs/reference/session_cli.md: Replaced placeholder with full ULID
- features/session_cli.feature: Consecutive When steps -> And
- features/steps/session_cli_steps.py: Summary panel asserts both IDs,
  8-char negative guard in table output, ULID capture scoped to table
  region with fixture verification, named-session absence check for
  second session
- CHANGELOG.md: Added [Unreleased] entry for the behavioral change

ISSUES CLOSED: #10970
2026-05-05 10:56:18 +00:00

709 lines
25 KiB
Python

"""Step definitions for the Session CLI feature."""
from __future__ import annotations
import json
import os
import re
import tempfile
from datetime import datetime
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
from ulid import ULID
from cleveragents.cli.commands import session as session_mod
from cleveragents.cli.commands.session import app as session_app
from cleveragents.domain.models.core.session import (
MessageRole,
Session,
SessionImportError,
SessionMessage,
SessionNotFoundError,
SessionTokenUsage,
)
_SESSION_ID = str(ULID())
_SESSION_ID_2 = str(ULID())
_ENVELOPE_KEYS = {"command", "status", "exit_code", "data", "timing", "messages"}
def _unwrap_envelope(parsed: Any) -> Any:
"""Return the ``data`` field if *parsed* is a spec envelope, else *parsed* as-is."""
if isinstance(parsed, dict) and _ENVELOPE_KEYS.issubset(parsed.keys()):
return parsed["data"]
return parsed
def _make_session(
*,
session_id: str | None = None,
actor_name: str | None = None,
messages: list[SessionMessage] | None = None,
) -> Session:
"""Create a test Session instance."""
return Session(
session_id=session_id or str(ULID()),
actor_name=actor_name,
namespace="local",
messages=messages or [],
token_usage=SessionTokenUsage(
input_tokens=100,
output_tokens=50,
estimated_cost=0.005,
),
created_at=datetime.now(),
updated_at=datetime.now(),
)
def _make_message(
role: MessageRole = MessageRole.USER,
content: str = "Hello",
sequence: int = 0,
) -> SessionMessage:
"""Create a test SessionMessage."""
return SessionMessage(
message_id=str(ULID()),
role=role,
content=content,
sequence=sequence,
timestamp=datetime.now(),
)
def _cleanup(context: Context) -> None:
"""Clean up temporary files."""
for path in getattr(context, "_cleanup_paths", []):
if os.path.exists(path):
os.unlink(path)
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a session CLI runner with mocked service")
def step_session_cli_runner(context: Context) -> None:
"""Set up CLI runner with a mocked session service."""
context.runner = CliRunner()
context.mock_service = MagicMock()
context._cleanup_paths: list[str] = []
# Default: create returns a session
default_session = _make_session(session_id=_SESSION_ID)
context.mock_service.create.return_value = default_session
# Patch the module-level service
session_mod._service = context.mock_service
def cleanup() -> None:
session_mod._service = None
_cleanup(context)
context.add_cleanup(cleanup)
# ---------------------------------------------------------------------------
# Create
# ---------------------------------------------------------------------------
@when("I run session CLI create with no arguments")
def step_create_no_args(context: Context) -> None:
context.result = context.runner.invoke(session_app, ["create"])
@when('I run session CLI create with --actor "{actor}"')
def step_create_with_actor(context: Context, actor: str) -> None:
session = _make_session(session_id=_SESSION_ID, actor_name=actor)
context.mock_service.create.return_value = session
context.result = context.runner.invoke(session_app, ["create", "--actor", actor])
@when("I run session CLI create with --format json")
def step_create_json(context: Context) -> None:
context.result = context.runner.invoke(session_app, ["create", "--format", "json"])
@then("the session CLI create should succeed")
def step_create_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# ---------------------------------------------------------------------------
# List
# ---------------------------------------------------------------------------
@given("there are no mocked sessions")
def step_no_sessions(context: Context) -> None:
context.mock_service.list.return_value = []
@given("there are mocked existing sessions")
def step_existing_sessions(context: Context) -> None:
sessions = [
_make_session(
session_id=_SESSION_ID,
actor_name="openai/gpt-4",
messages=[_make_message(sequence=0)],
),
_make_session(session_id=_SESSION_ID_2),
]
context.mock_service.list.return_value = sessions
@given("there are mocked existing named sessions")
def step_existing_named_sessions(context: Context) -> None:
"""Set up mocked sessions with names for Summary panel name-display test."""
sessions = [
_make_session(
session_id=_SESSION_ID,
actor_name="openai/gpt-4",
messages=[_make_message(sequence=0)],
),
_make_session(session_id=_SESSION_ID_2),
]
sessions[0].name = "weekly-planning"
sessions[1].name = "refactor-sprint"
context.mock_service.list.return_value = sessions
@when("I run session CLI list")
def step_list(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["list"], env={"COLUMNS": "200"}
)
@when("I run session CLI list with --format json")
def step_list_json(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["list", "--format", "json"], env={"COLUMNS": "200"}
)
@then("the session CLI should show all sessions in a table")
def step_list_shows_table(context: Context) -> None:
assert context.result.exit_code == 0
assert "Sessions" in context.result.output
@then("the session CLI rich table should display full session ULIDs")
def step_list_rich_table_full_ulids(context: Context) -> None:
"""Verify the Rich table displays the full 26-character session ULIDs."""
assert context.result.exit_code == 0
output = context.result.output
# Restrict assertion to the table region (before the Summary panel) so
# that a regression back to 8-char truncation is not masked by the full
# ULID appearing elsewhere (e.g. in the Summary panel).
summary_idx = output.find("Summary")
table_output = output[:summary_idx] if summary_idx != -1 else output
assert _SESSION_ID in table_output, (
f"Full ULID {_SESSION_ID} not found in Rich table output:\n"
f"{context.result.output}"
)
assert _SESSION_ID_2 in table_output, (
f"Full ULID {_SESSION_ID_2} not found in Rich table output:\n"
f"{context.result.output}"
)
# Negative guard: ensure the table contains full 26-character ULIDs,
# not the old 8-character truncated form. A standalone 8-char prefix
# (not as part of the full ULID) would indicate the [:8] slice was not
# removed.
table_without_full_ids = table_output.replace(_SESSION_ID, "").replace(
_SESSION_ID_2, ""
)
assert _SESSION_ID[:8] not in table_without_full_ids, (
f"Truncated 8-char ID {_SESSION_ID[:8]} found in Rich table output:\n"
f"{context.result.output}"
)
assert _SESSION_ID_2[:8] not in table_without_full_ids, (
f"Truncated 8-char ID {_SESSION_ID_2[:8]} found in Rich table output:\n"
f"{context.result.output}"
)
@then("the session CLI summary panel should contain full session ULIDs")
def step_list_summary_full_ulids(context: Context) -> None:
"""Verify the Summary panel shows full ULIDs for unnamed sessions."""
assert context.result.exit_code == 0
output = context.result.output
assert "Summary" in output, f"Summary panel not found in output:\n{output}"
# Extract the Summary panel region to avoid a false pass from the Rich
# table also containing the same full ULIDs.
summary_idx = output.find("Summary")
summary_output = output[summary_idx:]
# Both Most Recent and Oldest entries should display full ULIDs when
# sessions are unnamed. Asserting only one ID would allow a regression
# that re-introduced [:8] on one fallback path while keeping the other
# intact to pass the test undetected.
assert _SESSION_ID in summary_output, (
f"Full ULID {_SESSION_ID} not found in Summary panel:\n{output}"
)
assert _SESSION_ID_2 in summary_output, (
f"Full ULID {_SESSION_ID_2} not found in Summary panel:\n{output}"
)
@then("the session CLI summary panel should show session names")
def step_list_summary_shows_names(context: Context) -> None:
"""Verify the Summary panel shows session names (not ULIDs) for named sessions."""
assert context.result.exit_code == 0
output = context.result.output
summary_idx = output.find("Summary")
assert summary_idx != -1, f"Summary panel not found in output:\n{output}"
summary_output = output[summary_idx:]
assert "weekly-planning" in summary_output, (
f"'weekly-planning' not found in Summary panel:\n{output}"
)
assert "refactor-sprint" in summary_output, (
f"'refactor-sprint' not found in Summary panel:\n{output}"
)
# The Summary should NOT show the raw ULID when session names are present
assert _SESSION_ID not in summary_output, (
f"Full ULID {_SESSION_ID} unexpectedly found in Summary panel:\n{output}"
)
assert _SESSION_ID_2 not in summary_output, (
f"Full ULID {_SESSION_ID_2} unexpectedly found in Summary panel:\n{output}"
)
@when("I capture the first session full ULID from the output")
def step_capture_first_ulid(context: Context) -> None:
"""Parse the first session's full ULID from the output and store it
for subsequent steps (round-trip tell test)."""
assert context.result.exit_code == 0
output = context.result.output
# Restrict the search to the table region (before the Summary panel)
# so that a ULID appearing in the Summary panel is not accidentally
# captured as the "first" session ID.
summary_idx = output.find("Summary")
search_region = output[:summary_idx] if summary_idx != -1 else output
match = re.search(r"[0-9A-HJKMNP-TV-Z]{26}", search_region)
assert match is not None, (
f"No 26-character ULID found in table output:\n{search_region}"
)
context.full_session_id = match.group()
assert len(context.full_session_id) == 26, (
f"Parsed ULID '{context.full_session_id}' is not 26 characters"
)
# Sanity check: the captured ID should match one of the known fixture IDs
assert context.full_session_id in (_SESSION_ID, _SESSION_ID_2), (
f"Captured ULID '{context.full_session_id}' does not match any fixture ID"
)
@when('I run session CLI tell with the full session ID and prompt "{prompt}"')
def step_tell_with_stored_ulid(context: Context, prompt: str) -> None:
"""Run session tell using the full ULID stored from session list."""
session_id = context.full_session_id
context.mock_service.append_message.side_effect = [
_make_message(MessageRole.USER, prompt, 0),
_make_message(MessageRole.ASSISTANT, f"Acknowledged: {prompt}", 1),
]
context.result = context.runner.invoke(
session_app,
["tell", "--session", session_id, prompt],
)
# ---------------------------------------------------------------------------
# Show
# ---------------------------------------------------------------------------
@given("there is a mocked session with messages")
def step_session_with_messages(context: Context) -> None:
messages = [
_make_message(MessageRole.USER, "Hello", 0),
_make_message(MessageRole.ASSISTANT, "Hi there", 1),
]
session = _make_session(
session_id=_SESSION_ID,
actor_name="openai/gpt-4",
messages=messages,
)
context.mock_service.get.return_value = session
context.session_id = _SESSION_ID
@when("I run session CLI show with a valid session ID")
def step_show_valid(context: Context) -> None:
context.result = context.runner.invoke(session_app, ["show", context.session_id])
@when("I run session CLI show with --format json")
def step_show_json(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["show", context.session_id, "--format", "json"]
)
@when("I run session CLI show with an invalid session ID")
def step_show_invalid(context: Context) -> None:
context.mock_service.get.side_effect = SessionNotFoundError(
"Session 'INVALID' not found"
)
context.result = context.runner.invoke(session_app, ["show", "INVALID"])
@then("the session CLI show should succeed")
def step_show_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# ---------------------------------------------------------------------------
# Delete
# ---------------------------------------------------------------------------
@given("there is a mocked session to delete")
def step_session_to_delete(context: Context) -> None:
session = _make_session(session_id=_SESSION_ID)
context.mock_service.get.return_value = session
context.mock_service.delete.return_value = None
context.session_id = _SESSION_ID
@when("I run session CLI delete with --yes")
def step_delete_yes(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["delete", context.session_id, "--yes"]
)
@when("I run session CLI delete with a non-existent ID")
def step_delete_nonexistent(context: Context) -> None:
context.mock_service.get.side_effect = SessionNotFoundError("Session not found")
context.result = context.runner.invoke(
session_app, ["delete", "NONEXISTENT", "--yes"]
)
@then("the session CLI delete should succeed")
def step_delete_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# ---------------------------------------------------------------------------
# Export
# ---------------------------------------------------------------------------
@given("there is a mocked session for export")
def step_session_for_export(context: Context) -> None:
session = _make_session(session_id=_SESSION_ID)
context.mock_service.get.return_value = session
context.mock_service.export_session.return_value = session.as_export_dict()
context.session_id = _SESSION_ID
@given("there is an existing export file")
def step_existing_export_file(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".json")
with os.fdopen(fd, "w") as fh:
fh.write("{}")
context.existing_export_path = path
context._cleanup_paths.append(path)
@when("I run session CLI export with no output file")
def step_export_stdout(context: Context) -> None:
context.result = context.runner.invoke(session_app, ["export", context.session_id])
@when("I run session CLI export with --output to a temp file")
def step_export_to_file(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".json")
os.close(fd)
os.unlink(path) # Remove so export can create it
context.export_path = path
context._cleanup_paths.append(path)
context.result = context.runner.invoke(
session_app, ["export", context.session_id, "--output", path]
)
@when("I run session CLI export to an existing file without --force")
def step_export_no_force(context: Context) -> None:
context.result = context.runner.invoke(
session_app,
["export", context.session_id, "--output", context.existing_export_path],
)
@when("I run session CLI export to an existing file with --force")
def step_export_with_force(context: Context) -> None:
context.result = context.runner.invoke(
session_app,
[
"export",
context.session_id,
"--output",
context.existing_export_path,
"--force",
],
)
@when("I run session CLI export with a non-existent session ID")
def step_export_nonexistent(context: Context) -> None:
context.mock_service.export_session.side_effect = SessionNotFoundError(
"Session not found"
)
context.result = context.runner.invoke(session_app, ["export", "NONEXISTENT"])
@then("the session CLI export should succeed")
def step_export_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
@then("the exported file should exist")
def step_exported_file_exists(context: Context) -> None:
assert os.path.exists(context.export_path), (
f"Export file not found: {context.export_path}"
)
# ---------------------------------------------------------------------------
# Import
# ---------------------------------------------------------------------------
@given("there is a valid session export file")
def step_valid_export_file(context: Context) -> None:
session = _make_session(session_id=_SESSION_ID, actor_name="openai/gpt-4")
export_data = session.as_export_dict()
fd, path = tempfile.mkstemp(suffix=".json")
with os.fdopen(fd, "w") as fh:
json.dump(export_data, fh, default=str)
context.import_path = path
context._cleanup_paths.append(path)
imported = _make_session(session_id=str(ULID()), actor_name="openai/gpt-4")
context.mock_service.import_session.return_value = imported
@given("there is a corrupt session export file")
def step_corrupt_export_file(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".json")
with os.fdopen(fd, "w") as fh:
json.dump({"schema_version": "1.0", "bad": "data"}, fh)
context.corrupt_path = path
context._cleanup_paths.append(path)
context.mock_service.import_session.side_effect = SessionImportError(
"Checksum verification failed"
)
@given("there is an invalid JSON file")
def step_invalid_json_file(context: Context) -> None:
fd, path = tempfile.mkstemp(suffix=".json")
with os.fdopen(fd, "w") as fh:
fh.write("{not valid json")
context.invalid_json_path = path
context._cleanup_paths.append(path)
@when("I run session CLI import with the export file")
def step_import_valid(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["import", "--input", context.import_path]
)
@when("I run session CLI import with a non-existent file")
def step_import_nonexistent(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["import", "--input", "/tmp/nonexistent_session_file.json"]
)
@when("I run session CLI import with the corrupt file")
def step_import_corrupt(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["import", "--input", context.corrupt_path]
)
@when("I run session CLI import with the invalid JSON file")
def step_import_invalid_json(context: Context) -> None:
context.result = context.runner.invoke(
session_app, ["import", "--input", context.invalid_json_path]
)
@then("the session CLI import should succeed")
def step_import_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# ---------------------------------------------------------------------------
# Tell
# ---------------------------------------------------------------------------
@given("there is a mocked session for tell")
def step_session_for_tell(context: Context) -> None:
session = _make_session(session_id=_SESSION_ID)
context.mock_service.get.return_value = session
context.mock_service.append_message.side_effect = [
_make_message(MessageRole.USER, "Hello, world", 0),
_make_message(MessageRole.ASSISTANT, "Acknowledged: Hello, world", 1),
]
context.session_id = _SESSION_ID
@when('I run session CLI tell with a prompt "{prompt}"')
def step_tell_prompt(context: Context, prompt: str) -> None:
context.mock_service.append_message.side_effect = [
_make_message(MessageRole.USER, prompt, 0),
_make_message(MessageRole.ASSISTANT, f"Acknowledged: {prompt}", 1),
]
context.result = context.runner.invoke(
session_app,
["tell", "--session", context.session_id, prompt],
)
@when('I run session CLI tell with --actor "{actor}" and prompt "{prompt}"')
def step_tell_with_actor(context: Context, actor: str, prompt: str) -> None:
context.mock_service.append_message.side_effect = [
_make_message(MessageRole.USER, prompt, 0),
_make_message(
MessageRole.ASSISTANT,
f"[{actor}] Acknowledged: {prompt}",
1,
),
]
context.result = context.runner.invoke(
session_app,
["tell", "--session", context.session_id, "--actor", actor, prompt],
)
@when("I run session CLI tell to a non-existent session")
def step_tell_nonexistent(context: Context) -> None:
context.mock_service.append_message.side_effect = SessionNotFoundError(
"Session not found"
)
context.result = context.runner.invoke(
session_app,
["tell", "--session", "NONEXISTENT", "Hello"],
)
@then("the session CLI tell should succeed")
def step_tell_succeeds(context: Context) -> None:
assert context.result.exit_code == 0, (
f"Expected exit code 0, got {context.result.exit_code}: {context.result.output}"
)
# ---------------------------------------------------------------------------
# Generic assertions
# ---------------------------------------------------------------------------
@then('the session CLI output should contain "{text}"')
def step_output_contains(context: Context, text: str) -> None:
assert text in context.result.output, (
f"Expected '{text}' in output:\n{context.result.output}"
)
@then('the session CLI output should not contain "{text}"')
def step_output_not_contains(context: Context, text: str) -> None:
assert text not in context.result.output, (
f"Expected '{text}' NOT in output, but it was found:\n{context.result.output}"
)
@then("the session CLI output should be valid JSON")
def step_output_valid_json(context: Context) -> None:
try:
json.loads(context.result.output)
except json.JSONDecodeError as exc:
raise AssertionError(
f"Output is not valid JSON:\n{context.result.output}"
) from exc
@then('the session CLI JSON should contain "{key}"')
def step_json_contains_key(context: Context, key: str) -> None:
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
assert key in data, f"Key '{key}' not found in JSON: {data}"
@then("the session CLI JSON token usage should include counts")
def step_json_show_token_usage_counts(context: Context) -> None:
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
token_usage = data.get("token_usage")
assert isinstance(token_usage, dict), (
f"token_usage missing or not an object: {token_usage}"
)
_assert_token_usage_counts_are_ints(token_usage)
@then("the session CLI JSON list entries should match the documented contract")
def step_json_list_contract(context: Context) -> None:
parsed = json.loads(context.result.output)
data = _unwrap_envelope(parsed)
sessions = data.get("sessions")
assert isinstance(sessions, list), f"sessions missing or not a list: {sessions}"
for index, session in enumerate(sessions):
assert isinstance(session, dict), (
f"session entry at index {index} is not an object: {session!r}"
)
expected_keys = {"id", "name", "actor", "messages", "updated"}
actual_keys = set(session.keys())
missing = expected_keys - actual_keys
extra = actual_keys - expected_keys
assert not missing, (
f"session entry {index} missing keys {sorted(missing)}: {session!r}"
)
assert not extra, (
f"session entry {index} has undocumented keys {sorted(extra)}: {session!r}"
)
def _assert_token_usage_counts_are_ints(token_usage: dict[str, Any]) -> None:
input_tokens = token_usage.get("input_tokens")
output_tokens = token_usage.get("output_tokens")
assert isinstance(input_tokens, int), (
f"input_tokens should be int, got {input_tokens!r}"
)
assert isinstance(output_tokens, int), (
f"output_tokens should be int, got {output_tokens!r}"
)
assert input_tokens != "***REDACTED***", "input_tokens should not be redacted"
assert output_tokens != "***REDACTED***", "output_tokens should not be redacted"
@then("the session CLI should exit with error")
def step_exit_with_error(context: Context) -> None:
assert context.result.exit_code != 0, (
f"Expected non-zero exit code, got {context.result.exit_code}: "
f"{context.result.output}"
)