Compare commits

...

1 Commits

Author SHA1 Message Date
HAL9000 2a670f5964 fix(cli): add agents validation list command to validation CLI
Add a new 'list' subcommand to the 'agents validation' CLI group. The command
supports two modes:
  - Without --resource: Lists all registered validation definitions (tool_type=validation)
    in a formatted table or structured output formats (json/yaml/plain).
  - With --resource: Shows validation attachments scoped to that resource, optionally
    filtered by project or plan ID.

Includes BDD test suite covering empty/non-empty registry cases, rich/JSON formatting,
resource-scoped listing, project/plan filtering, and error handling paths.

ISSUES CLOSED: #8667
2026-05-08 14:05:24 +00:00
5 changed files with 504 additions and 0 deletions
+9
View File
@@ -5,6 +5,15 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## [Unreleased]
### Added
- **`agents validation list` command** (#8667): Added a new `list` subcommand to the
`agents validation` CLI group. Without ``--resource``, it lists all registered
validation definitions (`tool_type=validation`) in a formatted table. With
``--resource``, it shows validation attachments scoped to that resource, optionally
filtered by project or plan ID. Supports ``--format`` option for json/yaml/plain/table/rich
output formats.
- Fixed `ReactiveEventBus.emit()` exception handler to log the full exception
message (`str(exc)`) and enable traceback forwarding (`exc_info=True`).
Previously the handler logged only the exception type name (e.g.
+2
View File
@@ -10,6 +10,8 @@
# Details
* HAL 9000 has contributed the agents validation list CLI command (#8667): added a `list` subcommand to the `agents validation` group for discovering all registered validations and resource-scoped attachment queries with rich/structured output.
Below are some of the specific details of various contributions.
* Jeffrey Phillips Freeman has acted as Lead Developer, daily contributor, and Project Owner.
+289
View File
@@ -0,0 +1,289 @@
"""Step definitions for agents validation list CLI command (PR #8667).
Covers all code paths in the new ``list`` subcommand of
``cleveragents.cli.commands.validation``:
1. Listing all registered validations (rich table, json, yaml, plain)
2. Listing attachments for a specific resource (empty and non-empty cases)
3. Resource-scoped listing with project and plan filters
4. Error handling via CleverAgentsError exception path
"""
from __future__ import annotations
import json
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
from behave.runner import Context
from typer.testing import CliRunner
import cleveragents.cli.commands.validation as val_mod
_runner = CliRunner()
# ---------------------------------------------------------------------------
# Patch targets
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Background
# ---------------------------------------------------------------------------
@given("a validation list test runner with mocks")
def step_validation_list_background(context: Context) -> None:
"""Set up the CliRunner and initialize result tracking."""
context.val_list_runner = _runner
context.val_list_result = None
context.val_list_mock_service: MagicMock | None = None
# ---------------------------------------------------------------------------
# Service setup helpers
# ---------------------------------------------------------------------------
def _make_mock_validation(name: str, source: str = "custom", mode: str = "required", description: str = "") -> dict[str, Any]:
"""Create a mock validation spec dict."""
return {
"name": name,
"description": description or f"Mock validation {name}",
"source": source,
"tool_type": "validation",
"mode": mode,
}
def _make_mock_attachment(attachment_id: str, validation_name: str, resource_id: str, project_name: str | None = None, plan_id: str | None = None) -> dict[str, Any]:
"""Create a mock attachment dict."""
return {
"attachment_id": attachment_id,
"validation_name": validation_name,
"resource_id": resource_id,
"mode": "required",
"project_name": project_name,
"plan_id": plan_id,
"created_at": "2026-05-07T12:00:00Z",
}
def _patch_service(context: Context) -> Any:
"""Return a context manager that patches _get_tool_registry_service on the module."""
assert context.val_list_mock_service is not None, "Mock service must be set before invoking command"
return patch.object(val_mod, "_get_tool_registry_service", return_value=context.val_list_mock_service)
# ---------------------------------------------------------------------------
# Service setup steps (Given)
# ---------------------------------------------------------------------------
@given("the validation list service has no registered tools")
def step_setup_empty_validations(context: Context) -> None:
"""Prepare a mock service whose ``list_tools(tool_type="validation")`` returns []."""
svc = MagicMock()
svc.list_tools.return_value = []
# Empty for attachments too.
svc.list_validations_for_resource.return_value = []
context.val_list_mock_service = svc
@given("the validation list service has three mock validations registered")
def step_setup_three_validations(context: Context) -> None:
"""Prepare a mock service with three sample validation specs."""
svc = MagicMock()
mock_validations = [
_make_mock_validation("coverage-check", source="custom", mode="required", description="Check code coverage meets threshold"),
_make_mock_validation("lint-enforcement", source="builtin", mode="required", description="Enforce linting rules for all commits"),
_make_mock_validation("dependency-audit", source="custom", mode="informational", description="Audit dependencies for known vulnerabilities"),
]
svc.list_tools.return_value = mock_validations
context.val_list_mock_service = svc
@given('the validation list service has no attachments for resource "{resource}"')
def step_setup_empty_attachments(context: Context, resource: str) -> None:
"""Prepare a mock service whose ``list_validations_for_resource`` returns []."""
svc = MagicMock()
svc.list_tools.return_value = [] # still have validations for the other branch
svc.list_validations_for_resource.return_value = []
context.val_list_mock_service = svc
@given('the validation list service has two mock attachments for resource "{resource}"')
def step_setup_two_attachments(context: Context, resource: str) -> None:
"""Prepare a mock service with two sample attachment dicts."""
svc = MagicMock()
svc.list_tools.return_value = []
mock_attachments = [
_make_mock_attachment("01HQTESTATT1AAAAAAAAAAAAAA", "coverage-check", resource),
_make_mock_attachment("01HQTESTATT2BBBBBBBBBBBBBB", "lint-enforcement", resource),
]
svc.list_validations_for_resource.return_value = mock_attachments
context.val_list_mock_service = svc
@given('the validation list service has mock attachments for resource "{resource}" in project "{project}"')
def step_setup_project_attachments(context: Context, resource: str, project: str) -> None:
"""Prepare a mock service with attachments scoped to a project."""
svc = MagicMock()
svc.list_tools.return_value = []
mock_attachments = [
_make_mock_attachment("01HQTESTATT3CCCCCCCCCCCCCC", "coverage-check", resource, project_name=project),
]
svc.list_validations_for_resource.return_value = mock_attachments
context.val_list_mock_service = svc
@given('the validation list service has mock attachments for resource "{resource}" in plan "{plan_id}"')
def step_setup_plan_attachments(context: Context, resource: str, plan_id: str) -> None:
"""Prepare a mock service with attachments scoped to a plan."""
svc = MagicMock()
svc.list_tools.return_value = []
mock_attachments = [
_make_mock_attachment("01HQTESTATT4DDDDDDDDDDDDDD", "coverage-check", resource, plan_id=plan_id),
]
svc.list_validations_for_resource.return_value = mock_attachments
context.val_list_mock_service = svc
@given("the validation list service raises a CleverAgentsError")
def step_setup_error(context: Context) -> None:
"""Prepare a mock service that raises CleverAgentsError on any call."""
from cleveragents.core.exceptions import CleverAgentsError
exc = CleverAgentsError(message="Database connection failed")
svc = MagicMock()
svc.list_tools.side_effect = exc
svc.list_validations_for_resource.side_effect = exc
context.val_list_mock_service = svc
# ---------------------------------------------------------------------------
# Command invocation steps (When)
# ---------------------------------------------------------------------------
@when("I run validation list with default format")
def step_run_validation_list_default(context: Context) -> None:
"""Invoke the validation list command with no explicit --format flag."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list"])
@when('I run validation list with format "{fmt}"')
def step_run_validation_list_fmt(context: Context, fmt: str) -> None:
"""Invoke the validation list command with a specific format."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list", "--format", fmt])
@when('I run validation list with resource "{resource}"')
def step_run_validation_list_resource(context: Context, resource: str) -> None:
"""Invoke validation list with --resource flag."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list", "--resource", resource])
@when('I run validation list with resource "{resource}" and format "{fmt}"')
def step_run_validation_list_resource_fmt(context: Context, resource: str, fmt: str) -> None:
"""Invoke validation list with --resource and --format flags."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list", "--resource", resource, "--format", fmt])
@when('I run validation list with resource "{resource}" and project "{project}"')
def step_run_validation_list_resource_project(context: Context, resource: str, project: str) -> None:
"""Invoke validation list with --resource and --project flags."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list", "--resource", resource, "--project", project])
@when('I run validation list with resource "{resource}" and plan "{plan_id}"')
def step_run_validation_list_resource_plan(context: Context, resource: str, plan_id: str) -> None:
"""Invoke validation list with --resource and --plan flags."""
with _patch_service(context):
context.val_list_result = _runner.invoke(val_mod.app, ["list", "--resource", resource, "--plan", plan_id])
# ---------------------------------------------------------------------------
# Assertion steps (Then)
# ---------------------------------------------------------------------------
@then('the output should contain "{text}"')
def step_output_contains_text(context: Context, text: str) -> None:
assert context.val_list_result is not None
assert text in context.val_list_result.output, (
f"Expected '{text}' in output. Got: {context.val_list_result.output}"
)
@then("the output should contain 'No validation attachments found'")
def step_output_contains_no_attachments(context: Context) -> None:
assert context.val_list_result is not None
assert "No validation attachments found" in context.val_list_result.output
@then('the output contains "{text}"')
def step_output_contains_text(context: Context, text: str) -> None:
assert context.val_list_result is not None
assert text in context.val_list_result.output, (
f"Expected '{text}' in output. Got: {context.val_list_result.output}"
)
@then("the output should be valid json")
def step_output_is_valid_json(context: Context) -> None:
assert context.val_list_result is not None
try:
parsed = json.loads(context.val_list_result.output)
assert isinstance(parsed, list), "Expected a JSON array"
except (json.JSONDecodeError, AssertionError) as e:
raise AssertionError(f"Output is not valid JSON: {e}. Output: {context.val_list_result.output}")
@then("the output should contain a list of validation dictionaries")
def step_output_contains_validation_dicts(context: Context) -> None:
assert context.val_list_result is not None
parsed = json.loads(context.val_list_result.output)
assert isinstance(parsed, list) and len(parsed) > 0
for item in parsed:
assert "name" in item or "attachment_id" in item
@then("the output should contain attachment dictionaries")
def step_output_contains_attachment_dicts(context: Context) -> None:
assert context.val_list_result is not None
parsed = json.loads(context.val_list_result.output)
assert isinstance(parsed, list) and len(parsed) > 0
for item in parsed:
assert "attachment_id" in item
@then("the command should succeed with exit code {code:d}")
def step_command_succeeds(context: Context, code: int) -> None:
assert context.val_list_result is not None
assert context.val_list_result.exit_code == code, (
f"Expected exit code {code}, got {context.val_list_result.exit_code}. Output: {context.val_list_result.output}"
)
@then("the output should not contain a non-rich format")
def step_output_is_rich_table(context: Context) -> None:
assert context.val_list_result is not None
# Rich table uses unicode box-drawing characters, plain/json do not
@then("the command should abort with error")
def step_command_aborts_error(context: Context) -> None:
assert context.val_list_result is not None
assert context.val_list_result.exit_code != 0, (
f"Expected non-zero exit code, got {context.val_list_result.exit_code}. Output: {context.val_list_result.output}"
)
assert "Error:" in context.val_list_result.output, (
f"Expected 'Error:' in output. Got: {context.val_list_result.output}"
)
+88
View File
@@ -0,0 +1,88 @@
Feature: Agents validation list CLI command (PR #8667)
As a developer
I want to list all registered validations and resource-scoped attachments
So that I can discover what validations are available and inspect their attachments
Background:
Given a validation list test runner with mocks
# ──────────────────────────────────────────
# Scenario 1: List all validations (empty registry)
# ──────────────────────────────────────────
Scenario: Validation list returns no results for empty registry
Given the validation list service has no registered tools
When I run validation list with default format
Then the output should contain "No validations found"
And the command should succeed with exit code 0
# ──────────────────────────────────────────
# Scenario 2: List all validations (non-empty registry)
# ──────────────────────────────────────────
Scenario: Validation list shows registered validations in rich table
Given the validation list service has three mock validations registered
When I run validation list with default format
Then the output should contain "Registered Validations"
And the output should contain "coverage-check"
And the output should not contain a non-rich format
Scenario: Validation list shows registrations in json format
Given the validation list service has three mock validations registered
When I run validation list with format "json"
Then the output should be valid json
And the output should contain a list of validation dictionaries
# ──────────────────────────────────────────
# Scenario 3: List attachments for a resource (empty)
# ──────────────────────────────────────────
Scenario: Validation list with --resource shows no attachments when none exist
Given the validation list service has no attachments for resource "git-checkout/my-repo"
When I run validation list with resource "git-checkout/my-repo"
Then the output should contain "No validation attachments found"
And the command should succeed with exit code 0
# ──────────────────────────────────────────
# Scenario 4: List attachments for a resource (with data)
# ──────────────────────────────────────────
Scenario: Validation list --resource shows attachments in rich table
Given the validation list service has two mock attachments for resource "git-checkout/my-repo"
When I run validation list with resource "git-checkout/my-repo"
Then the output should contain "Validation Attachments on git-checkout/my-repo"
And the output should not contain a non-rich format
Scenario: Validation list --resource shows attachments in json format
Given the validation list service has two mock attachments for resource "git-checkout/my-repo"
When I run validation list with resource "git-checkout/my-repo" and format "json"
Then the output should be valid json
And the output should contain attachment dictionaries
# ──────────────────────────────────────────
# Scenario 5: Validation list --resource with project scope
# ──────────────────────────────────────────
Scenario: Validation list --resource scoped by project shows project filter in title
Given the validation list service has mock attachments for resource "git-checkout/my-repo" in project "myapp"
When I run validation list with resource "git-checkout/my-repo" and project "myapp"
Then the output should contain "[myapp]"
# ──────────────────────────────────────────
# Scenario 6: Validation list --resource with plan scope
# ──────────────────────────────────────────
Scenario: Validation list --resource scoped by plan shows plan filter in title
Given the validation list service has mock attachments for resource "git-checkout/my-repo" in plan "01HQTESTPLAN1234567890ABCDEF"
When I run validation list with resource "git-checkout/my-repo" and plan "01HQTESTPLAN1234567890ABCDEF"
Then the output should contain "[01HQTESTPLAN1234567890ABCDEF]"
# ──────────────────────────────────────────
# Scenario 7: Error handling
# ──────────────────────────────────────────
Scenario: Validation list aborts when service throws CleverAgentsError
Given the validation list service raises a CleverAgentsError
When I run validation list with default format
Then the command should abort with error
And the output should mention "Error:"
+116
View File
@@ -8,6 +8,7 @@ subtypes) and their lifecycle attachments to resources.
| Command | Description |
|--------------------------------|------------------------------------------|
| ``agents validation add`` | Register validation from YAML config |
| ``agents validation list`` | List validations or resource attachments |
| ``agents validation attach`` | Attach validation to a resource |
| ``agents validation detach`` | Detach a validation attachment |
@@ -55,6 +56,7 @@ import typer
import yaml
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from cleveragents.cli.bootstrap import ensure_cli_database_bootstrapped
from cleveragents.cli.formatting import OutputFormat, format_output
@@ -415,3 +417,117 @@ def detach(
except CleverAgentsError as exc:
console.print(f"[red]Error:[/red] {exc.message}")
raise typer.Abort() from exc
@app.command("list")
def list_validations(
resource: Annotated[
str | None,
typer.Option("--resource", "-r", help="List attachments for a specific resource"),
] = None,
project: Annotated[
str | None,
typer.Option("--project", "-p", help="Project scope (used with --resource)"),
] = None,
plan_id: Annotated[
str | None,
typer.Option("--plan", help="Plan ID scope (used with --resource)"),
] = None,
fmt: Annotated[
str,
typer.Option("--format", "-f", help=_FORMAT_HELP),
] = "rich",
) -> None:
"""List validations or resource attachments.
Without ``--resource``, this lists all registered validation definitions.
With ``--resource`` set, it shows the validations attached to that
specific resource (optionally scoped by project / plan).
Examples:
agents validation list
agents validation list --resource git-checkout/my-repo
agents validation list --project myapp --resource git-checkout/my-repo
agents validation list --format json
"""
try:
service = _get_tool_registry_service()
if resource is not None:
# List attachments for a specific resource.
attachments = service.list_validations_for_resource(
resource_id=resource,
project_name=project,
plan_id=plan_id,
)
if not attachments:
console.print("[yellow]No validation attachments found.[/yellow]")
return
if fmt != OutputFormat.RICH.value:
data = [dict(_attachment_dict(a)) for a in attachments]
console.print(format_output(data, fmt))
return
table = Table(
title=f"Validation Attachments on {resource}"
+ (f" [{project}]" if project else "")
+ (f" [{plan_id}]" if plan_id else ""),
)
table.add_column("Attachment ID", style="cyan")
table.add_column("Validation", style="blue")
table.add_column("Resource", style="magenta")
table.add_column("Mode", style="yellow")
table.add_column("Created At", style="dim")
for att in attachments:
d = _attachment_dict(att)
table.add_row(
str(d.get("attachment_id", "")),
str(d.get("validation_name", "")),
str(d.get("resource_id", "")),
str(d.get("mode", "required")),
str(d.get("created_at", "")),
)
console.print(table)
else:
# List all registered validation definitions.
validations = service.list_tools(tool_type="validation")
if not validations:
console.print("[yellow]No validations found.[/yellow]")
console.print("Register one with 'agents validation add --config <file>'")
return
if fmt != OutputFormat.RICH.value:
data = [dict(_validation_spec_dict(v)) for v in validations]
console.print(format_output(data, fmt))
return
# Rich table for registered validations.
table = Table(title=f"Registered Validations ({len(validations)} total)")
table.add_column("Name", style="cyan")
table.add_column("Source", style="magenta")
table.add_column("Mode", style="yellow")
table.add_column("Description", style="dim")
for tool in validations:
spec = _validation_spec_dict(tool)
desc = str(spec.get("description", ""))
if len(desc) > 50:
desc = desc[:47] + "..."
table.add_row(
str(spec.get("name", "")),
str(spec.get("source", "")),
str(spec.get("mode", "required")),
desc,
)
console.print(table)
except CleverAgentsError as exc:
console.print(f"[red]Error:[/red] {exc.message}")
raise typer.Abort() from exc