From bd7ce2ab8804e92a476e6e4e9c6e704ff6302353 Mon Sep 17 00:00:00 2001 From: Jeffrey Phillips Freeman Date: Tue, 7 Apr 2026 09:07:27 +0000 Subject: [PATCH] docs: add showcase example for config and automation profiles Adds a complete end-to-end walkthrough of the config management and automation profile CLI commands, verified by the UAT system with real command outputs. Covers: - config list (all settings, filtered by pattern) - config get (single value + verbose 5-level resolution chain) - config set (global/project/local scope) - automation-profile list (all profiles + regex filter) - automation-profile show (built-in profiles: supervised, full-auto, manual) - automation-profile add (custom profile from YAML with guards) - automation-profile remove (with --yes flag) Also updates examples.json index with the new entry. --- .../config-and-automation-profiles.md | 738 ++++++++++++++++++ docs/showcase/examples.json | 24 + 2 files changed, 762 insertions(+) create mode 100644 docs/showcase/cli-tools/config-and-automation-profiles.md diff --git a/docs/showcase/cli-tools/config-and-automation-profiles.md b/docs/showcase/cli-tools/config-and-automation-profiles.md new file mode 100644 index 000000000..31f43012d --- /dev/null +++ b/docs/showcase/cli-tools/config-and-automation-profiles.md @@ -0,0 +1,738 @@ +# Managing Config and Automation Profiles in CleverAgents + +## Overview + +CleverAgents ships with a rich configuration system and a set of **automation +profiles** that control how much autonomy the agent has when executing plans. +This example walks through the complete workflow: inspecting configuration +values, understanding the five-level resolution chain, switching automation +profiles, and creating a custom profile with guard constraints. + +## Prerequisites + +- CleverAgents installed (`pip install cleveragents`) +- Python 3.12 or higher + +## What You'll Learn + +- How to **list all configuration settings** and filter by key pattern +- How to **get a single config value** with its resolution chain +- How to **set a config value** at global, project, or local scope +- How to **list and inspect automation profiles** (built-in and custom) +- How to **create a custom automation profile** from a YAML file with guard + constraints +- How to **remove a custom profile** when it is no longer needed + +--- + +## Part 1: Configuration Management + +### Step 1: List all configuration settings + +```bash +$ agents config list +``` + +**Expected Output (truncated):** +``` + Configuration (106 settings) +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━┓ +┃ Key ┃ Value ┃ Source ┃ Modified ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━┩ +│ core.automation-profile │ supervised │ default │ │ +│ core.log.level │ DEBUG │ local │ yes │ +│ plan.concurrency │ 4 │ default │ │ +│ sandbox.strategy │ git_worktree │ default │ │ +│ provider.anthropic.api-key │ **** │ env_var │ yes │ +└────────────────────────────┴────────────────────────────┴─────────┴──────────┘ +``` + +**What's Happening:** + +The table shows every registered configuration key, its current effective +value, the **source** that supplied the value (one of `cli_flag`, `env_var`, +`local`, `project`, `global`, or `default`), and whether the value differs +from the built-in default. Secret values (API keys, tokens, passwords) are +automatically masked as `****`. + +--- + +### Step 2: Filter settings by key pattern + +```bash +$ agents config list "plan.*" --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": [ + {"key": "plan.budget.per-plan", "value": null, "source": "default", "modified": false}, + {"key": "plan.budget.per-session", "value": null, "source": "default", "modified": false}, + {"key": "plan.budget.warn-threshold", "value": 0.8, "source": "default", "modified": false}, + {"key": "plan.concurrency", "value": 4, "source": "default", "modified": false}, + {"key": "plan.max-child-depth", "value": 5, "source": "default", "modified": false}, + {"key": "plan.tool.max-calls-per-step","value": 25, "source": "default", "modified": false}, + {"key": "plan.tool.max-retries", "value": 3, "source": "default", "modified": false}, + {"key": "plan.tool.retry-backoff", "value": "exponential", "source": "default", "modified": false}, + {"key": "sandbox.checkpoint.max-per-plan", "value": 50, "source": "default", "modified": false} + ], + "timing": {"duration_ms": 1}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +The positional argument to `config list` is a **regex pattern** applied to key +names. `plan.*` matches all keys starting with `plan.` as well as +`sandbox.checkpoint.max-per-plan` (because the regex is searched anywhere in +the key). Use `--format json` to get machine-readable output suitable for +scripting. + +--- + +### Step 3: Get a single config value + +```bash +$ agents config get core.log.level --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "key": "core.log.level", + "value": "DEBUG", + "source": "local", + "type": "str" + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +`config get` returns the **effective value** for a single key along with the +source that supplied it. Here `source: "local"` means the value was read from +`config.local.toml` in the project root — the highest-priority file-based +source. + +--- + +### Step 4: Inspect the full five-level resolution chain + +```bash +$ agents config get core.automation-profile --verbose --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "key": "core.automation-profile", + "value": "supervised", + "source": "default", + "type": "str", + "resolution_chain": [ + {"source": "cli_flag", "value": null}, + {"source": "env_var", "value": null, "env_name": "CLEVERAGENTS_AUTOMATION_PROFILE"}, + {"source": "local", "value": null, "path": "/app/config.local.toml"}, + {"source": "project", "value": null, "path": "/app/config.toml"}, + {"source": "global", "value": null, "path": "/home/user/.cleveragents/config.toml"}, + {"source": "default", "value": "supervised"} + ] + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +The `--verbose` flag exposes the **five-level precedence chain** (highest → +lowest priority): + +| Priority | Source | Description | +|----------|-------------|--------------------------------------------------| +| 1 (high) | `cli_flag` | `--` flag passed directly on the command line | +| 2 | `env_var` | Environment variable (e.g. `CLEVERAGENTS_AUTOMATION_PROFILE`) | +| 3 | `local` | `config.local.toml` in the project root (gitignored) | +| 4 | `project` | `config.toml` in the project root (committed) | +| 5 | `global` | `~/.cleveragents/config.toml` (user-wide) | +| 6 (low) | `default` | Built-in default value | + +The first non-null value wins. Here every level is null except `default`, so +`supervised` is the effective value. + +--- + +### Step 5: Set a config value at global scope + +```bash +$ agents config set plan.concurrency 8 --scope global --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "key": "plan.concurrency", + "value": 8, + "previous_value": null, + "source": "config_file", + "scope": "global" + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +`config set` writes the value to `~/.cleveragents/config.toml` (global scope). +The response includes `previous_value` so you can see what was there before. +Use `--scope project` to write to `config.toml` or `--scope local` to write to +`config.local.toml` (the local override file, typically gitignored). + +Restore the default: + +```bash +$ agents config set plan.concurrency 4 --scope global --format json +``` + +--- + +## Part 2: Automation Profiles + +Automation profiles control **how much autonomy** the agent has when executing +plans. Each threshold is a float in `[0.0, 1.0]` where `0.0` means fully +automatic and `1.0` means human approval is always required. + +### Step 6: List all automation profiles + +```bash +$ agents automation-profile list --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "profiles": [ + {"name": "auto", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Fully automatic except apply"}, + {"name": "cautious", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Probabilistic gates on most actions"}, + {"name": "ci", "source": "built-in", "select_tool": 0.0, "sandbox": true, "description": "Designed for CI pipelines"}, + {"name": "full-auto", "source": "built-in", "select_tool": 0.0, "sandbox": false, "description": "No gates, no sandbox, no checkpoints"}, + {"name": "manual", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Human approves every action"}, + {"name": "review", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Human reviews before apply"}, + {"name": "supervised", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Human reviews strategy and execution"}, + {"name": "trusted", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Auto for most, human for apply and revert"} + ], + "summary": {"built_in": 8, "custom": 0, "total": 8} + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +Eight built-in profiles ship with every CleverAgents installation. The +`select_tool` field is the threshold for automatic tool selection (the +Execute→Apply gate). `sandbox: true` means the profile requires a git worktree +sandbox for safe execution. + +--- + +### Step 7: Filter profiles by regex + +```bash +$ agents automation-profile list "^(manual|supervised|auto)$" --format json +``` + +**Expected Output:** +```json +{ + "data": { + "profiles": [ + {"name": "auto", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Fully automatic except apply"}, + {"name": "manual", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Human approves every action"}, + {"name": "supervised", "source": "built-in", "select_tool": 1.0, "sandbox": true, "description": "Human reviews strategy and execution"} + ], + "summary": {"built_in": 3, "custom": 0, "total": 3} + } +} +``` + +**What's Happening:** + +The positional argument to `automation-profile list` is a **regex pattern** +applied to profile names. This is useful when you have many custom profiles and +want to find a specific subset. + +--- + +### Step 8: Inspect a built-in profile in detail + +```bash +$ agents automation-profile show supervised --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "name": "supervised", + "description": "Human reviews strategy and execution", + "source": "built-in", + "schema_version": "1.0", + "phase_transitions": { + "decompose_task": 0.0, + "create_tool": 1.0, + "select_tool": 1.0 + }, + "decision_automation": { + "edit_code": 0.0, + "execute_command": 1.0 + }, + "self_repair": { + "create_file": 1.0, + "delete_content": 1.0, + "access_network": 1.0, + "modify_config": 0.0, + "approve_plan": 1.0 + }, + "execution_controls": { + "install_dependency": 1.0, + "require_sandbox": true, + "require_checkpoints": true, + "allow_unsafe_tools": false + }, + "guards": null + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +The `show` command returns the full profile with thresholds grouped into four +semantic categories: + +| Category | Controls | +|-----------------------|------------------------------------------------------------| +| `phase_transitions` | Whether the agent auto-advances between plan phases | +| `decision_automation` | Whether the agent auto-edits code or runs commands | +| `self_repair` | Whether the agent auto-creates files, reverts, etc. | +| `execution_controls` | Sandbox, checkpoints, and unsafe tool permissions | + +In `supervised`, `decompose_task: 0.0` means task decomposition is fully +automatic, but `create_tool: 1.0` means the agent always asks before creating +new tools. + +--- + +### Step 9: Compare profiles — `full-auto` vs `manual` + +```bash +$ agents automation-profile show full-auto --format json +``` + +**Expected Output (key fields):** +```json +{ + "data": { + "name": "full-auto", + "description": "No gates, no sandbox, no checkpoints", + "phase_transitions": {"decompose_task": 0.0, "create_tool": 0.0, "select_tool": 0.0}, + "decision_automation": {"edit_code": 0.0, "execute_command": 0.0}, + "self_repair": {"create_file": 0.0, "delete_content": 0.0, "access_network": 0.0, "modify_config": 0.0, "approve_plan": 0.0}, + "execution_controls": {"install_dependency": 0.0, "require_sandbox": false, "require_checkpoints": false, "allow_unsafe_tools": true} + } +} +``` + +```bash +$ agents automation-profile show manual --format json +``` + +**Expected Output (key fields):** +```json +{ + "data": { + "name": "manual", + "description": "Human approves every action", + "phase_transitions": {"decompose_task": 1.0, "create_tool": 1.0, "select_tool": 1.0}, + "decision_automation": {"edit_code": 1.0, "execute_command": 1.0}, + "self_repair": {"create_file": 1.0, "delete_content": 1.0, "access_network": 1.0, "modify_config": 1.0, "approve_plan": 1.0}, + "execution_controls": {"install_dependency": 1.0, "require_sandbox": true, "require_checkpoints": true, "allow_unsafe_tools": false} + } +} +``` + +**What's Happening:** + +`full-auto` sets every threshold to `0.0` (fully automatic) and disables the +sandbox and checkpoints — maximum speed, minimum safety. `manual` sets every +threshold to `1.0` (always ask) and enables all safety controls — maximum +safety, minimum autonomy. These are the two extremes; the other six built-in +profiles sit between them. + +--- + +## Part 3: Custom Automation Profiles + +### Step 10: Create a custom profile YAML file + +Save the following as `my-profile.yaml`: + +```yaml +# Custom profile: acme/cautious +# Based on the built-in 'cautious' profile with additional guard +# constraints for controlled environments. + +name: acme/cautious +description: Cautious profile with guard constraints +schema_version: "1.0" + +# Task-type confidence thresholds (0.0 = auto, 1.0 = human approval) +decompose_task: 0.7 +create_tool: 0.7 +select_tool: 1.0 + +edit_code: 0.6 +execute_command: 0.8 + +create_file: 0.7 +delete_content: 0.8 +access_network: 0.9 + +install_dependency: 0.7 +modify_config: 0.0 +approve_plan: 0.6 + +# Safety requirements +safety: + require_sandbox: true + require_checkpoints: true + allow_unsafe_tools: false + +# Guard constraints — hard limits enforced at runtime +guards: + max_tool_calls_per_step: 10 + max_total_cost: 5.0 + tool_denylist: + - shell_exec + - file_delete + require_approval_for_writes: true + require_approval_for_apply: true +``` + +**Key points about the YAML format:** + +- `name` must be a bare name (`my-profile`) or namespaced (`namespace/name`) +- `schema_version` must be `"1.0"` (quoted string) +- All threshold fields are floats in `[0.0, 1.0]` +- `guards` is optional — omit it for a profile without hard limits +- `tool_denylist` blocks specific tools from being called + +--- + +### Step 11: Register the custom profile + +```bash +$ agents automation-profile add --config my-profile.yaml --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "name": "acme/cautious", + "description": "Cautious profile with guard constraints", + "source": "custom", + "schema_version": "1.0", + "phase_transitions": { + "decompose_task": 0.7, + "create_tool": 0.7, + "select_tool": 1.0 + }, + "decision_automation": { + "edit_code": 0.6, + "execute_command": 0.8 + }, + "self_repair": { + "create_file": 0.7, + "delete_content": 0.8, + "access_network": 0.9, + "modify_config": 0.0, + "approve_plan": 0.6 + }, + "execution_controls": { + "install_dependency": 0.7, + "require_sandbox": true, + "require_checkpoints": true, + "allow_unsafe_tools": false + }, + "guards": { + "max_tool_calls_per_step": 10, + "max_total_cost": 5.0, + "tool_allowlist": null, + "tool_denylist": ["shell_exec", "file_delete"], + "require_approval_for_writes": true, + "require_approval_for_apply": true + } + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +The profile is validated against the schema (Pydantic model) and persisted to +the CleverAgents database. The `source` field is `"custom"` to distinguish it +from built-in profiles. The `guards` block is stored and enforced at runtime — +if a plan step tries to call more than 10 tools or exceeds $5.00 in cost, the +guard fires. + +--- + +### Step 12: Verify the custom profile appears in the list + +```bash +$ agents automation-profile list --format json +``` + +**Expected Output (summary section):** +```json +{ + "data": { + "profiles": [ + {"name": "acme/cautious", "source": "custom", "select_tool": 1.0, "sandbox": true, "description": "Cautious profile with guard constraints"}, + ... + ], + "summary": {"built_in": 8, "custom": 1, "total": 9} + } +} +``` + +The `summary.custom` count is now `1` and the profile appears in the list +sorted alphabetically alongside the built-in profiles. + +--- + +### Step 13: Activate the custom profile + +```bash +$ agents config set core.automation-profile acme/cautious --scope global --format json +``` + +**Expected Output:** +```json +{ + "data": { + "key": "core.automation-profile", + "value": "acme/cautious", + "previous_value": null, + "source": "config_file", + "scope": "global" + } +} +``` + +All subsequent plan executions will now use the `acme/cautious` profile. + +--- + +### Step 14: Remove the custom profile + +```bash +$ agents automation-profile remove acme/cautious --yes --format json +``` + +**Expected Output:** +```json +{ + "command": "", + "status": "ok", + "exit_code": 0, + "data": { + "name": "acme/cautious", + "description": "Cautious profile with guard constraints", + "source": "custom", + "removed": true, + ... + }, + "timing": {"duration_ms": 0}, + "messages": [{"level": "ok", "text": "ok"}] +} +``` + +**What's Happening:** + +`--yes` skips the interactive confirmation prompt — useful in scripts. Built-in +profiles cannot be removed; attempting to do so returns an error. The response +includes the full profile data with `removed: true` so you can confirm what was +deleted. + +--- + +## Complete Interaction Log + +
+Click to see the full verified command sequence + +``` +# 1. List all 106 config settings (rich table) +$ agents config list + +# 2. Filter to plan.* settings in JSON +$ agents config list "plan.*" --format json +# → 9 keys returned including plan.concurrency=4, plan.max-child-depth=5 + +# 3. Get a single value +$ agents config get core.log.level --format json +# → {"key": "core.log.level", "value": "DEBUG", "source": "local", "type": "str"} + +# 4. Verbose resolution chain +$ agents config get core.automation-profile --verbose --format json +# → resolution_chain shows 6 levels; default wins with "supervised" + +# 5. Set a value at global scope +$ agents config set plan.concurrency 8 --scope global --format json +# → {"key": "plan.concurrency", "value": 8, "previous_value": null, "scope": "global"} + +# 6. Restore the default +$ agents config set plan.concurrency 4 --scope global --format json +# → {"key": "plan.concurrency", "value": 4, "previous_value": 8, "scope": "global"} + +# 7. List all automation profiles +$ agents automation-profile list --format json +# → 8 built-in profiles: auto, cautious, ci, full-auto, manual, review, supervised, trusted + +# 8. Filter profiles by regex +$ agents automation-profile list "^(manual|supervised|auto)$" --format json +# → 3 profiles returned + +# 9. Show supervised profile details +$ agents automation-profile show supervised --format json +# → full threshold breakdown in 4 categories + +# 10. Show full-auto (all 0.0, no sandbox) +$ agents automation-profile show full-auto --format json + +# 11. Show manual (all 1.0, sandbox+checkpoints) +$ agents automation-profile show manual --format json + +# 12. Add custom profile from YAML +$ agents automation-profile add --config my-profile.yaml --format json +# → source: "custom", guards with max_tool_calls_per_step=10 + +# 13. Verify it appears in list +$ agents automation-profile list --format json +# → summary: {built_in: 8, custom: 1, total: 9} + +# 14. Activate the custom profile +$ agents config set core.automation-profile acme/cautious --scope global --format json + +# 15. Remove the custom profile +$ agents automation-profile remove acme/cautious --yes --format json +# → removed: true +``` +
+ +--- + +## Key Takeaways + +- **Config uses a five-level precedence chain**: `cli_flag` > `env_var` > + `local` > `project` > `global` > `default`. Use `--verbose` to see which + level wins for any key. +- **Three file scopes**: `--scope global` writes to `~/.cleveragents/config.toml`, + `--scope project` to `config.toml`, `--scope local` to `config.local.toml` + (typically gitignored for per-developer overrides). +- **Secret values are always masked**: API keys, tokens, and passwords appear + as `****` in all output formats unless `--show-secrets` is passed. +- **Eight built-in profiles** cover the full autonomy spectrum from `manual` + (all thresholds 1.0, always ask) to `full-auto` (all thresholds 0.0, never + ask). The `ci` profile is optimised for unattended pipeline execution. +- **Custom profiles use `namespace/name`** format (e.g. `acme/cautious`) and + can include `guards` for hard runtime limits on tool calls, cost, and + write operations. +- **Profiles are activated via config**: set `core.automation-profile` to the + profile name to make it the default for all plan executions. + +## Try It Yourself + +```bash +# See which config values differ from defaults +$ agents config list --filter-values "." | grep "yes" + +# Inspect the CI profile for pipeline use +$ agents automation-profile show ci --format yaml + +# Create a read-only profile (no file writes, no network) +$ cat > readonly.yaml << 'EOF' +name: team/readonly +description: Read-only analysis profile +schema_version: "1.0" +decompose_task: 0.0 +create_tool: 1.0 +select_tool: 1.0 +edit_code: 1.0 +execute_command: 1.0 +create_file: 1.0 +delete_content: 1.0 +access_network: 1.0 +install_dependency: 1.0 +modify_config: 1.0 +approve_plan: 1.0 +safety: + require_sandbox: true + require_checkpoints: true + allow_unsafe_tools: false +EOF +$ agents automation-profile add --config readonly.yaml + +# Use it for a single session via env var +$ CLEVERAGENTS_AUTOMATION_PROFILE=team/readonly agents plan list +``` + +## Related Examples + +- See [`output-format-flags.md`](output-format-flags.md) for the full guide to + `--format json/yaml/plain/table/rich` +- See `docs/showcase/cli-tools/` for more CLI tool examples + +--- +*This example was automatically generated and verified by the CleverAgents UAT system.* +*Feature area: Config and automation profiles | Test cycle: 1 | Generated: 2026-04-07* + +--- +**Automated by CleverAgents Bot** +Supervisor: UAT Testing | Agent: uat-tester diff --git a/docs/showcase/examples.json b/docs/showcase/examples.json index a9f1908f0..3fc5cd317 100644 --- a/docs/showcase/examples.json +++ b/docs/showcase/examples.json @@ -68,6 +68,30 @@ "educational_value": "high", "generated_by": "uat-tester", "generated_at": "2026-04-07" + }, + { + "title": "Managing Config and Automation Profiles", + "category": "cli-tools", + "path": "cli-tools/config-and-automation-profiles.md", + "feature": "Config and automation profiles", + "commands": [ + "agents config list", + "agents config list \"plan.*\" --format json", + "agents config get core.log.level --format json", + "agents config get core.automation-profile --verbose --format json", + "agents config set plan.concurrency 8 --scope global --format json", + "agents automation-profile list --format json", + "agents automation-profile list \"^(manual|supervised|auto)$\" --format json", + "agents automation-profile show supervised --format json", + "agents automation-profile show full-auto --format json", + "agents automation-profile show manual --format json", + "agents automation-profile add --config my-profile.yaml --format json", + "agents automation-profile remove acme/cautious --yes --format json" + ], + "complexity": "intermediate", + "educational_value": "high", + "generated_by": "uat-tester", + "generated_at": "2026-04-07" } ], "categories": {