diff --git a/benchmarks/security_audit_bench.py b/benchmarks/security_audit_bench.py new file mode 100644 index 000000000..21f80d96f --- /dev/null +++ b/benchmarks/security_audit_bench.py @@ -0,0 +1,136 @@ +"""ASV benchmarks for SEC7 - Audit logging overhead. + +Measures the performance of recording, querying, and pruning +audit log entries to establish baselines for write overhead. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from cleveragents.application.services.audit_service import AuditService +from cleveragents.config.settings import Settings +from cleveragents.infrastructure.database.models import AuditLogModel, Base + + +def _make_service() -> AuditService: + """Create a service backed by in-memory SQLite.""" + Settings._instance = None + settings = Settings() + engine = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + return AuditService(settings=settings, session=session) + + +def _seed_entries(service: AuditService, count: int) -> None: + """Insert *count* entries for benchmarking.""" + for i in range(count): + service.record( + event_type="plan_applied", + plan_id=f"plan-{i}", + project_name="bench-project", + details={"index": i}, + ) + + +class TimeRecordSingle: + """Benchmark recording a single audit event.""" + + timeout = 30 + + def setup(self) -> None: + self.service = _make_service() + + def teardown(self) -> None: + pass + + def time_record_single_event(self) -> None: + self.service.record( + event_type="plan_applied", + plan_id="bench-plan", + project_name="bench-project", + details={"files_changed": 5}, + ) + + +class TimeRecordBatch: + """Benchmark recording a batch of audit events.""" + + timeout = 60 + + def setup(self) -> None: + self.service = _make_service() + + def teardown(self) -> None: + pass + + def time_record_100_events(self) -> None: + for i in range(100): + self.service.record( + event_type="plan_applied", + plan_id=f"plan-{i}", + details={"index": i}, + ) + + +class TimeQueryEntries: + """Benchmark querying audit log entries.""" + + timeout = 30 + + def setup(self) -> None: + self.service = _make_service() + _seed_entries(self.service, 500) + + def teardown(self) -> None: + pass + + def time_list_all(self) -> None: + self.service.list_entries(limit=100) + + def time_list_by_plan(self) -> None: + self.service.list_entries(plan_id="plan-42") + + def time_list_by_project(self) -> None: + self.service.list_entries(project_name="bench-project") + + def time_list_by_type(self) -> None: + self.service.list_entries(event_type="plan_applied") + + def time_get_single(self) -> None: + self.service.get_entry(250) + + def time_count(self) -> None: + self.service.count() + + +class TimePrune: + """Benchmark pruning old audit entries.""" + + timeout = 30 + + def setup(self) -> None: + self.service = _make_service() + # Insert entries backdated to 60 days ago + old_time = (datetime.now(tz=UTC) - timedelta(days=60)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + for i in range(200): + row = AuditLogModel( + event_type="plan_applied", + plan_id=f"plan-{i}", + details="{}", + created_at=old_time, + ) + self.service._session.add(row) + self.service._session.commit() + + def teardown(self) -> None: + pass + + def time_prune_200_entries(self) -> None: + self.service.prune(retention_days=30) diff --git a/docs/reference/audit_logging.md b/docs/reference/audit_logging.md new file mode 100644 index 000000000..cb82c5740 --- /dev/null +++ b/docs/reference/audit_logging.md @@ -0,0 +1,113 @@ +# Audit Logging + +CleverAgents records security-relevant operations in an `audit_log` table +for compliance, post-hoc analysis, and diagnostics. + +## Event Types + +| Event Type | Trigger | Details Captured | +|---|---|---| +| `plan_applied` | `agents plan apply` | Plan ID, project, files changed, validation results | +| `plan_cancelled` | `agents plan cancel` | Plan ID, reason, resources released | +| `resource_modified` | Tool write during Execute | Resource ID, modification type, tool name | +| `correction_applied` | `agents plan correct` | Correction ID, original decision, mode, guidance | +| `config_changed` | `agents config set` | Key, old/new values (masked if secret) | +| `entity_deleted` | `agents delete` | Entity type, entity name, cascade effects | +| `session_created` | `agents session create` | Session ID, actor name | + +## CLI Commands + +### `agents audit list` + +List audit entries with optional filters. + +```bash +# List recent entries +agents audit list + +# Filter by plan +agents audit list --plan 01HXYZ123 + +# Filter by project and time range +agents audit list --project myproject --since 2026-02-01T00:00:00 + +# Filter by event type with limit +agents audit list --type plan_applied --limit 10 +``` + +### `agents audit show ` + +Display a single entry with full metadata and JSON details. + +```bash +agents audit show 42 +``` + +### `agents audit prune` + +Delete entries older than the retention period. + +```bash +# Use configured retention (default: keep indefinitely) +agents audit prune --yes + +# Override retention to 90 days +agents audit prune --days 90 --yes +``` + +### `agents audit count` + +Display the total number of audit log entries. + +```bash +agents audit count +``` + +## Configuration + +| Setting | Environment Variable | Default | Description | +|---|---|---|---| +| `audit_retention_days` | `CLEVERAGENTS_AUDIT_RETENTION_DAYS` | `0` | Days to retain entries. `0` = keep indefinitely. | + +## Database Schema + +```sql +CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_type TEXT NOT NULL, + plan_id TEXT, + project_name TEXT, + actor_name TEXT, + user_identity TEXT, + details TEXT NOT NULL, -- JSON + created_at TEXT NOT NULL +); + +CREATE INDEX idx_audit_event ON audit_log(event_type); +CREATE INDEX idx_audit_plan ON audit_log(plan_id) WHERE plan_id IS NOT NULL; +CREATE INDEX idx_audit_created ON audit_log(created_at); +``` + +## Retention Policy + +By default, audit logs are retained indefinitely (`audit_retention_days=0`) +for compliance purposes. Use `agents audit prune` to manually remove old +entries, or set a non-zero retention to enable periodic pruning. + +## Recording Events + +The `AuditService.record()` method writes events: + +```python +from cleveragents.application.services.audit_service import AuditService +from cleveragents.config.settings import get_settings + +service = AuditService(settings=get_settings()) +service.record( + event_type="plan_applied", + plan_id="01HXYZ123", + project_name="my-project", + actor_name="openai/gpt-4o", + details={"files_changed": 3, "validation_passed": True}, +) +``` diff --git a/features/security_audit.feature b/features/security_audit.feature new file mode 100644 index 000000000..0722f19c9 --- /dev/null +++ b/features/security_audit.feature @@ -0,0 +1,181 @@ +Feature: SEC7 - Audit logging for apply + As a security-conscious developer + I want all security-relevant operations recorded in an audit log + So that I have a compliance trail of plan applies, cancellations, and changes + + # ── Recording events ────────────────────────────────────────── + + Scenario: Record a plan_applied event + Given a fresh audit service + When I record a "plan_applied" event with plan_id "plan-001" and project "myproject" + Then the audit log should contain 1 entry + And the entry event_type should be "plan_applied" + And the entry plan_id should be "plan-001" + And the entry project_name should be "myproject" + + Scenario: Record a plan_cancelled event + Given a fresh audit service + When I record a "plan_cancelled" event with plan_id "plan-002" and details reason "user request" + Then the audit log should contain 1 entry + And the entry event_type should be "plan_cancelled" + And the entry details should contain key "reason" + + Scenario: Record a resource_modified event + Given a fresh audit service + When I record a "resource_modified" event with details resource_id "res-001" and tool "file_write" + Then the audit log should contain 1 entry + And the entry event_type should be "resource_modified" + And the entry details key "tool" should be "file_write" + + Scenario: Record a correction_applied event + Given a fresh audit service + When I record a "correction_applied" event with actor "openai/gpt-4o" + Then the entry actor_name should be "openai/gpt-4o" + + Scenario: Record a config_changed event + Given a fresh audit service + When I record a "config_changed" event with user "hamza" + Then the entry user_identity should be "hamza" + + Scenario: Record an entity_deleted event + Given a fresh audit service + When I record a "entity_deleted" event with details entity_type "resource" and entity_name "old-res" + Then the entry details key "entity_type" should be "resource" + And the entry details key "entity_name" should be "old-res" + + Scenario: Record event with empty details + Given a fresh audit service + When I record a "plan_applied" event with no details + Then the audit log should contain 1 entry + And the entry details should be empty + + Scenario: Record multiple events + Given a fresh audit service + When I record 5 "plan_applied" events + Then the audit log should contain 5 entries + + # ── Querying / filtering ────────────────────────────────────── + + Scenario: List entries filtered by plan_id + Given a fresh audit service with mixed events + When I list entries with plan filter "plan-A" + Then all returned entries should have plan_id "plan-A" + + Scenario: List entries filtered by project + Given a fresh audit service with mixed events + When I list entries with project filter "project-X" + Then all returned entries should have project_name "project-X" + + Scenario: List entries filtered by event_type + Given a fresh audit service with mixed events + When I list entries with type filter "plan_cancelled" + Then all returned entries should have event_type "plan_cancelled" + + Scenario: List entries filtered by since timestamp + Given a fresh audit service with old and new events + When I list entries since "2099-01-01T00:00:00" + Then no entries should be returned + + Scenario: List entries with limit + Given a fresh audit service with 20 events + When I list entries with limit 5 + Then exactly 5 entries should be returned + + Scenario: List entries returns newest first + Given a fresh audit service + When I record a "plan_applied" event with plan_id "first" and project "p" + And I record a "plan_applied" event with plan_id "second" and project "p" + And I list all entries + Then the first entry should have plan_id "second" + + # ── Show single entry ───────────────────────────────────────── + + Scenario: Show entry by ID + Given a fresh audit service + When I record a "plan_applied" event with plan_id "plan-show" and project "proj" + And I fetch the entry by its ID + Then the fetched entry should have event_type "plan_applied" + And the fetched entry should have plan_id "plan-show" + + Scenario: Show non-existent entry returns None + Given a fresh audit service + When I fetch entry with ID 99999 + Then no entry should be returned + + # ── Count ────────────────────────────────────────────────────── + + Scenario: Count returns total entries + Given a fresh audit service + When I record 7 "config_changed" events + Then the count should be 7 + + Scenario: Count on empty log returns zero + Given a fresh audit service + Then the count should be 0 + + # ── Retention / pruning ──────────────────────────────────────── + + Scenario: Prune deletes entries older than retention days + Given a fresh audit service with entries from 60 days ago + When I prune with retention 30 days + Then the audit log should be empty + + Scenario: Prune keeps recent entries + Given a fresh audit service + When I record a "plan_applied" event with plan_id "recent" and project "p" + And I prune with retention 30 days + Then the audit log should contain 1 entry + + Scenario: Prune with zero retention keeps everything + Given a fresh audit service with entries from 60 days ago + When I prune with retention 0 days + Then the audit log should not be empty + + Scenario: Prune returns count of deleted entries + Given a fresh audit service with 10 entries from 90 days ago + When I prune with retention 30 days + Then the prune result should be 10 + + # ── Settings ─────────────────────────────────────────────────── + + Scenario: Default audit retention is zero (keep indefinitely) + Given default settings + Then audit_retention_days should be 0 + + Scenario: Audit retention can be configured via env var + Given settings with audit_retention_days 90 + Then audit_retention_days should be 90 + + # ── Data class serialization ─────────────────────────────────── + + Scenario: AuditLogEntry as_dict returns all fields + Given a fresh audit service + When I record a "plan_applied" event with plan_id "p-1" and project "proj-1" + And I serialize the entry to dict + Then the dict should contain keys "id" "event_type" "plan_id" "project_name" "created_at" "details" + + Scenario: AuditLogEntry details contain structured data + Given a fresh audit service + When I record a "plan_applied" event with details files_changed 3 and validation_passed true + And I serialize the entry to dict + Then the dict details key "files_changed" should be 3 + + # ── Edge cases ───────────────────────────────────────────────── + + Scenario: Record event with all nullable fields as None + Given a fresh audit service + When I record a "plan_applied" event with all nullable fields as None + Then the entry plan_id should be None + And the entry project_name should be None + And the entry actor_name should be None + And the entry user_identity should be None + + Scenario: Details with non-serializable values use str fallback + Given a fresh audit service + When I record a "plan_applied" event with datetime in details + Then the audit log should contain 1 entry + And the entry details should contain key "timestamp" + + Scenario: Service requires Settings instance + When I create an AuditService with invalid settings + Then a TypeError should be raised for audit service diff --git a/features/steps/security_audit_steps.py b/features/steps/security_audit_steps.py new file mode 100644 index 000000000..020d96c70 --- /dev/null +++ b/features/steps/security_audit_steps.py @@ -0,0 +1,459 @@ +"""Step definitions for SEC7 - Audit logging for apply. + +Tests the audit service: recording, querying, filtering, retention, +pruning, and data class serialization. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from behave import given, then, when +from behave.runner import Context +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from cleveragents.application.services.audit_service import AuditService +from cleveragents.config.settings import Settings +from cleveragents.infrastructure.database.models import AuditLogModel, Base + +# ── Helpers ─────────────────────────────────────────────────────── + + +def _make_settings(**overrides: object) -> Settings: + """Create a Settings instance with optional field overrides.""" + Settings._instance = None + base = Settings() + if overrides: + return base.model_copy(update=overrides) + return base + + +def _fresh_service(settings: Settings | None = None) -> AuditService: + """Create an AuditService backed by an in-memory SQLite database.""" + if settings is None: + settings = _make_settings(database_url="sqlite:///:memory:") + engine = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine)() + return AuditService(settings=settings, session=session) + + +def _insert_old_entry( + service: AuditService, + days_ago: int, + event_type: str = "plan_applied", +) -> None: + """Insert an entry with a manually backdated created_at.""" + old_time = (datetime.now(tz=UTC) - timedelta(days=days_ago)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + row = AuditLogModel( + event_type=event_type, + details="{}", + created_at=old_time, + ) + service._session.add(row) + service._session.commit() + + +# ── Given ───────────────────────────────────────────────────────── + + +@given("a fresh audit service") +def step_fresh_service(context: Context) -> None: + context.service = _fresh_service() + context.entry = None + context.entries = [] + context.prune_result = None + + +@given("a fresh audit service with mixed events") +def step_fresh_with_mixed(context: Context) -> None: + context.service = _fresh_service() + svc = context.service + svc.record(event_type="plan_applied", plan_id="plan-A", project_name="project-X") + svc.record(event_type="plan_applied", plan_id="plan-A", project_name="project-Y") + svc.record(event_type="plan_cancelled", plan_id="plan-B", project_name="project-X") + svc.record(event_type="config_changed", project_name="project-Z") + context.entry = None + context.entries = [] + + +@given("a fresh audit service with old and new events") +def step_fresh_with_old_new(context: Context) -> None: + context.service = _fresh_service() + _insert_old_entry(context.service, days_ago=100) + context.service.record(event_type="plan_applied", plan_id="new") + context.entry = None + context.entries = [] + + +@given("a fresh audit service with {count:d} events") +def step_fresh_with_count(context: Context, count: int) -> None: + context.service = _fresh_service() + for i in range(count): + context.service.record(event_type="plan_applied", plan_id=f"plan-{i}") + context.entry = None + context.entries = [] + + +@given("a fresh audit service with entries from {days:d} days ago") +def step_fresh_with_old(context: Context, days: int) -> None: + context.service = _fresh_service() + _insert_old_entry(context.service, days_ago=days) + context.entry = None + context.entries = [] + context.prune_result = None + + +@given("a fresh audit service with {count:d} entries from {days:d} days ago") +def step_fresh_count_old(context: Context, count: int, days: int) -> None: + context.service = _fresh_service() + for _ in range(count): + _insert_old_entry(context.service, days_ago=days) + context.entry = None + context.entries = [] + context.prune_result = None + + +@given("default settings") +def step_default_settings(context: Context) -> None: + context.settings = _make_settings() + + +@given("settings with audit_retention_days {days:d}") +def step_settings_retention(context: Context, days: int) -> None: + context.settings = _make_settings(audit_retention_days=days) + + +# ── When ────────────────────────────────────────────────────────── + + +@when( + 'I record a "{event_type}" event with plan_id "{plan_id}" and project "{project}"' +) +def step_record_plan_project( + context: Context, event_type: str, plan_id: str, project: str +) -> None: + context.entry = context.service.record( + event_type=event_type, plan_id=plan_id, project_name=project + ) + + +@when( + 'I record a "{event_type}" event with plan_id "{plan_id}" ' + 'and details reason "{reason}"' +) +def step_record_with_reason( + context: Context, event_type: str, plan_id: str, reason: str +) -> None: + context.entry = context.service.record( + event_type=event_type, + plan_id=plan_id, + details={"reason": reason}, + ) + + +@when( + 'I record a "{event_type}" event with details resource_id ' + '"{resource_id}" and tool "{tool}"' +) +def step_record_resource_modified( + context: Context, event_type: str, resource_id: str, tool: str +) -> None: + context.entry = context.service.record( + event_type=event_type, + details={"resource_id": resource_id, "tool": tool}, + ) + + +@when('I record a "{event_type}" event with actor "{actor}"') +def step_record_with_actor(context: Context, event_type: str, actor: str) -> None: + context.entry = context.service.record(event_type=event_type, actor_name=actor) + + +@when('I record a "{event_type}" event with user "{user}"') +def step_record_with_user(context: Context, event_type: str, user: str) -> None: + context.entry = context.service.record(event_type=event_type, user_identity=user) + + +@when( + 'I record a "{event_type}" event with details entity_type ' + '"{entity_type}" and entity_name "{entity_name}"' +) +def step_record_entity_deleted( + context: Context, event_type: str, entity_type: str, entity_name: str +) -> None: + context.entry = context.service.record( + event_type=event_type, + details={"entity_type": entity_type, "entity_name": entity_name}, + ) + + +@when('I record a "{event_type}" event with no details') +def step_record_no_details(context: Context, event_type: str) -> None: + context.entry = context.service.record(event_type=event_type) + + +@when('I record {count:d} "{event_type}" events') +def step_record_multiple(context: Context, count: int, event_type: str) -> None: + for i in range(count): + context.entry = context.service.record( + event_type=event_type, plan_id=f"plan-{i}" + ) + + +@when('I list entries with plan filter "{plan_id}"') +def step_list_by_plan(context: Context, plan_id: str) -> None: + context.entries = context.service.list_entries(plan_id=plan_id) + + +@when('I list entries with project filter "{project}"') +def step_list_by_project(context: Context, project: str) -> None: + context.entries = context.service.list_entries(project_name=project) + + +@when('I list entries with type filter "{event_type}"') +def step_list_by_type(context: Context, event_type: str) -> None: + context.entries = context.service.list_entries(event_type=event_type) + + +@when('I list entries since "{since}"') +def step_list_since(context: Context, since: str) -> None: + context.entries = context.service.list_entries(since=since) + + +@when("I list entries with limit {limit:d}") +def step_list_with_limit(context: Context, limit: int) -> None: + context.entries = context.service.list_entries(limit=limit) + + +@when("I list all entries") +def step_list_all(context: Context) -> None: + context.entries = context.service.list_entries() + + +@when("I fetch the entry by its ID") +def step_fetch_by_id(context: Context) -> None: + context.fetched = context.service.get_entry(context.entry.id) + + +@when("I fetch entry with ID {audit_id:d}") +def step_fetch_nonexistent(context: Context, audit_id: int) -> None: + context.fetched = context.service.get_entry(audit_id) + + +@when("I prune with retention {days:d} days") +def step_prune(context: Context, days: int) -> None: + context.prune_result = context.service.prune(retention_days=days) + + +@when("I serialize the entry to dict") +def step_serialize(context: Context) -> None: + context.entry_dict = context.entry.as_dict() + + +@when('I record a "{event_type}" event with all nullable fields as None') +def step_record_all_none(context: Context, event_type: str) -> None: + context.entry = context.service.record(event_type=event_type) + + +@when('I record a "{event_type}" event with datetime in details') +def step_record_datetime_details(context: Context, event_type: str) -> None: + context.entry = context.service.record( + event_type=event_type, + details={"timestamp": datetime.now(tz=UTC)}, + ) + + +@when( + 'I record a "{event_type}" event with details files_changed ' + "{count:d} and validation_passed true" +) +def step_record_structured_details( + context: Context, event_type: str, count: int +) -> None: + context.entry = context.service.record( + event_type=event_type, + details={"files_changed": count, "validation_passed": True}, + ) + + +@when("I create an AuditService with invalid settings") +def step_create_invalid(context: Context) -> None: + try: + AuditService(settings="not-settings") # type: ignore[arg-type] + context.error = None + except TypeError as exc: + context.error = exc + + +# ── Then ────────────────────────────────────────────────────────── + + +@then("the audit log should contain {count:d} entry") +@then("the audit log should contain {count:d} entries") +def step_check_count(context: Context, count: int) -> None: + assert context.service.count() == count, ( + f"Expected {count}, got {context.service.count()}" + ) + + +@then('the entry event_type should be "{expected}"') +def step_check_event_type(context: Context, expected: str) -> None: + assert context.entry.event_type == expected + + +@then('the entry plan_id should be "{expected}"') +def step_check_plan_id(context: Context, expected: str) -> None: + assert context.entry.plan_id == expected + + +@then('the entry project_name should be "{expected}"') +def step_check_project(context: Context, expected: str) -> None: + assert context.entry.project_name == expected + + +@then('the entry actor_name should be "{expected}"') +def step_check_actor(context: Context, expected: str) -> None: + assert context.entry.actor_name == expected + + +@then('the entry user_identity should be "{expected}"') +def step_check_user(context: Context, expected: str) -> None: + assert context.entry.user_identity == expected + + +@then('the entry details should contain key "{key}"') +def step_check_details_key(context: Context, key: str) -> None: + assert key in context.entry.details, f"Key {key!r} not in {context.entry.details}" + + +@then('the entry details key "{key}" should be "{expected}"') +def step_check_details_value(context: Context, key: str, expected: str) -> None: + assert str(context.entry.details[key]) == expected + + +@then("the entry details should be empty") +def step_check_details_empty(context: Context) -> None: + assert context.entry.details == {} or context.entry.details is None + + +@then('all returned entries should have plan_id "{expected}"') +def step_all_plan_id(context: Context, expected: str) -> None: + assert len(context.entries) > 0 + for e in context.entries: + assert e.plan_id == expected, f"Entry {e.id} has plan_id={e.plan_id}" + + +@then('all returned entries should have project_name "{expected}"') +def step_all_project(context: Context, expected: str) -> None: + assert len(context.entries) > 0 + for e in context.entries: + assert e.project_name == expected + + +@then('all returned entries should have event_type "{expected}"') +def step_all_event_type(context: Context, expected: str) -> None: + assert len(context.entries) > 0 + for e in context.entries: + assert e.event_type == expected + + +@then("no entries should be returned") +def step_no_entries(context: Context) -> None: + assert len(context.entries) == 0 + + +@then("exactly {count:d} entries should be returned") +def step_exact_count(context: Context, count: int) -> None: + assert len(context.entries) == count, ( + f"Expected {count}, got {len(context.entries)}" + ) + + +@then('the first entry should have plan_id "{expected}"') +def step_first_plan_id(context: Context, expected: str) -> None: + assert context.entries[0].plan_id == expected + + +@then('the fetched entry should have event_type "{expected}"') +def step_fetched_event_type(context: Context, expected: str) -> None: + assert context.fetched is not None + assert context.fetched.event_type == expected + + +@then('the fetched entry should have plan_id "{expected}"') +def step_fetched_plan_id(context: Context, expected: str) -> None: + assert context.fetched is not None + assert context.fetched.plan_id == expected + + +@then("no entry should be returned") +def step_no_entry(context: Context) -> None: + assert context.fetched is None + + +@then("the count should be {expected:d}") +def step_count_is(context: Context, expected: int) -> None: + assert context.service.count() == expected + + +@then("the audit log should be empty") +def step_log_empty(context: Context) -> None: + assert context.service.count() == 0 + + +@then("the audit log should not be empty") +def step_log_not_empty(context: Context) -> None: + assert context.service.count() > 0 + + +@then("the prune result should be {expected:d}") +def step_prune_result(context: Context, expected: int) -> None: + assert context.prune_result == expected + + +@then("audit_retention_days should be {expected:d}") +def step_retention_days(context: Context, expected: int) -> None: + assert context.settings.audit_retention_days == expected + + +@then('the dict should contain keys "{k1}" "{k2}" "{k3}" "{k4}" "{k5}" "{k6}"') +def step_dict_keys( + context: Context, k1: str, k2: str, k3: str, k4: str, k5: str, k6: str +) -> None: + d = context.entry_dict + for key in (k1, k2, k3, k4, k5, k6): + assert key in d, f"Key {key!r} not in dict" + + +@then('the dict details key "{key}" should be {expected:d}') +def step_dict_details_int(context: Context, key: str, expected: int) -> None: + assert context.entry_dict["details"][key] == expected + + +@then("the entry plan_id should be None") +def step_plan_id_none(context: Context) -> None: + assert context.entry.plan_id is None + + +@then("the entry project_name should be None") +def step_project_none(context: Context) -> None: + assert context.entry.project_name is None + + +@then("the entry actor_name should be None") +def step_actor_none(context: Context) -> None: + assert context.entry.actor_name is None + + +@then("the entry user_identity should be None") +def step_user_none(context: Context) -> None: + assert context.entry.user_identity is None + + +@then("a TypeError should be raised for audit service") +def step_type_error(context: Context) -> None: + assert isinstance(context.error, TypeError) diff --git a/robot/security_audit.robot b/robot/security_audit.robot new file mode 100644 index 000000000..4317742af --- /dev/null +++ b/robot/security_audit.robot @@ -0,0 +1,80 @@ +*** Settings *** +Documentation SEC7 - Audit logging integration smoke tests +Library OperatingSystem +Library Process + +*** Variables *** +${AUDIT_SERVICE} ${CURDIR}/../src/cleveragents/application/services/audit_service.py +${AUDIT_CLI} ${CURDIR}/../src/cleveragents/cli/commands/audit.py +${AUDIT_MODEL} ${CURDIR}/../src/cleveragents/infrastructure/database/models.py +${AUDIT_DOC} ${CURDIR}/../docs/reference/audit_logging.md + +*** Test Cases *** +Audit Service Module Exists + [Documentation] Verify the audit service module is present + File Should Exist ${AUDIT_SERVICE} + +Audit CLI Module Exists + [Documentation] Verify the audit CLI module is present + File Should Exist ${AUDIT_CLI} + +Audit Log Model Defined + [Documentation] Verify AuditLogModel is defined in models.py + ${content}= Get File ${AUDIT_MODEL} + Should Contain ${content} class AuditLogModel + Should Contain ${content} audit_log + Should Contain ${content} event_type + Should Contain ${content} plan_id + Should Contain ${content} details + Should Contain ${content} created_at + +Audit Service Has Record Method + [Documentation] Verify the audit service has a record method + ${content}= Get File ${AUDIT_SERVICE} + Should Contain ${content} def record( + Should Contain ${content} event_type + +Audit Service Has List Method + [Documentation] Verify the audit service has a list_entries method + ${content}= Get File ${AUDIT_SERVICE} + Should Contain ${content} def list_entries( + Should Contain ${content} plan_id + Should Contain ${content} project_name + Should Contain ${content} since + +Audit Service Has Prune Method + [Documentation] Verify the audit service has a prune method + ${content}= Get File ${AUDIT_SERVICE} + Should Contain ${content} def prune( + Should Contain ${content} retention_days + +Audit CLI Has List Command + [Documentation] Verify the audit CLI has a list command + ${content}= Get File ${AUDIT_CLI} + Should Contain ${content} name="list" + Should Contain ${content} --plan + Should Contain ${content} --project + Should Contain ${content} --since + +Audit CLI Has Show Command + [Documentation] Verify the audit CLI has a show command + ${content}= Get File ${AUDIT_CLI} + Should Contain ${content} name="show" + Should Contain ${content} audit_id + +Audit CLI Has Prune Command + [Documentation] Verify the audit CLI has a prune command + ${content}= Get File ${AUDIT_CLI} + Should Contain ${content} name="prune" + Should Contain ${content} --days + Should Contain ${content} --yes + +Audit Documentation Exists + [Documentation] Verify the audit logging reference doc exists + File Should Exist ${AUDIT_DOC} + ${content}= Get File ${AUDIT_DOC} + Should Contain ${content} Audit Logging + Should Contain ${content} agents audit list + Should Contain ${content} agents audit show + Should Contain ${content} agents audit prune + Should Contain ${content} audit_retention_days diff --git a/src/cleveragents/application/services/audit_service.py b/src/cleveragents/application/services/audit_service.py new file mode 100644 index 000000000..b208c30cc --- /dev/null +++ b/src/cleveragents/application/services/audit_service.py @@ -0,0 +1,225 @@ +"""Audit logging service (SEC7). + +Records and queries security-relevant events such as plan applies, +cancellations, resource modifications, corrections, config changes, +and entity deletions. Supports filtering by plan, project, time +range, and event type, plus a configurable retention/prune policy. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + +from cleveragents.config.settings import Settings +from cleveragents.infrastructure.database.models import AuditLogModel, Base + +__all__ = [ + "AuditLogEntry", + "AuditService", +] + +# ── Domain data class ──────────────────────────────────────────── + + +@dataclass(slots=True) +class AuditLogEntry: + """A single audit log entry returned by queries.""" + + id: int + event_type: str + plan_id: str | None + project_name: str | None + actor_name: str | None + user_identity: str | None + details: dict[str, Any] + created_at: str + + def as_dict(self) -> dict[str, Any]: + """Serialize for CLI output.""" + return { + "id": self.id, + "event_type": self.event_type, + "plan_id": self.plan_id, + "project_name": self.project_name, + "actor_name": self.actor_name, + "user_identity": self.user_identity, + "details": self.details, + "created_at": self.created_at, + } + + +# ── Service ────────────────────────────────────────────────────── + + +class AuditService: + """Records and queries audit log entries. + + Args: + settings: Application settings (for DB URL and retention). + session: Optional pre-built SQLAlchemy session (for testing). + """ + + def __init__( + self, + settings: Settings, + session: Session | None = None, + ) -> None: + if not isinstance(settings, Settings): + raise TypeError("settings must be a Settings instance") + self._settings = settings + if session is not None: + self._session = session + else: + engine = create_engine(settings.database_url, echo=False) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + self._session = factory() + + # ── Record ─────────────────────────────────────────────────── + + def record( + self, + *, + event_type: str, + plan_id: str | None = None, + project_name: str | None = None, + actor_name: str | None = None, + user_identity: str | None = None, + details: dict[str, Any] | None = None, + ) -> AuditLogEntry: + """Write an audit event to the database. + + Returns: + The created :class:`AuditLogEntry` with its assigned ``id``. + """ + now = datetime.now(tz=UTC).strftime("%Y-%m-%dT%H:%M:%S.%f") + details_json = json.dumps(details or {}, default=str) + + row = AuditLogModel( + event_type=event_type, + plan_id=plan_id, + project_name=project_name, + actor_name=actor_name, + user_identity=user_identity, + details=details_json, + created_at=now, + ) + self._session.add(row) + self._session.commit() + self._session.refresh(row) + + return self._row_to_entry(row) + + # ── Query ──────────────────────────────────────────────────── + + def list_entries( + self, + *, + plan_id: str | None = None, + project_name: str | None = None, + event_type: str | None = None, + since: str | None = None, + limit: int = 100, + ) -> list[AuditLogEntry]: + """Query audit log entries with optional filters. + + Args: + plan_id: Filter by exact plan_id. + project_name: Filter by exact project name. + event_type: Filter by event type. + since: ISO-8601 timestamp; only entries after this time. + limit: Maximum number of entries to return. + + Returns: + List of matching entries, most recent first. + """ + query = self._session.query(AuditLogModel) + + if plan_id is not None: + query = query.filter(AuditLogModel.plan_id == plan_id) + if project_name is not None: + query = query.filter(AuditLogModel.project_name == project_name) + if event_type is not None: + query = query.filter(AuditLogModel.event_type == event_type) + if since is not None: + query = query.filter(AuditLogModel.created_at >= since) + + query = query.order_by(AuditLogModel.created_at.desc()).limit(limit) + + return [self._row_to_entry(row) for row in query.all()] + + def get_entry(self, audit_id: int) -> AuditLogEntry | None: + """Fetch a single audit entry by ID. + + Returns: + The entry, or ``None`` if not found. + """ + row = ( + self._session.query(AuditLogModel) + .filter(AuditLogModel.id == audit_id) + .first() + ) + if row is None: + return None + return self._row_to_entry(row) + + def count(self) -> int: + """Return the total number of audit log entries.""" + return self._session.query(AuditLogModel).count() + + # ── Retention / prune ──────────────────────────────────────── + + def prune(self, retention_days: int | None = None) -> int: + """Delete audit entries older than *retention_days*. + + Args: + retention_days: Override the configured retention. If + ``None``, uses ``settings.audit_retention_days``. + A value of ``0`` means "keep everything" (no pruning). + + Returns: + Number of rows deleted. + """ + days = ( + retention_days + if retention_days is not None + else self._settings.audit_retention_days + ) + if days <= 0: + return 0 + cutoff = (datetime.now(tz=UTC) - timedelta(days=days)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + result = ( + self._session.query(AuditLogModel) + .filter(AuditLogModel.created_at < cutoff) + .delete(synchronize_session="fetch") + ) + self._session.commit() + return result + + # ── Helpers ─────────────────────────────────────────────────── + + @staticmethod + def _row_to_entry(row: AuditLogModel) -> AuditLogEntry: + """Convert a SQLAlchemy model row to a domain data class.""" + try: + details = json.loads(row.details) if row.details else {} # type: ignore[arg-type] + except (json.JSONDecodeError, TypeError): + details = {} + return AuditLogEntry( + id=row.id, # type: ignore[arg-type] + event_type=row.event_type, # type: ignore[arg-type] + plan_id=row.plan_id, # type: ignore[arg-type] + project_name=row.project_name, # type: ignore[arg-type] + actor_name=row.actor_name, # type: ignore[arg-type] + user_identity=row.user_identity, # type: ignore[arg-type] + details=details, + created_at=row.created_at, # type: ignore[arg-type] + ) diff --git a/src/cleveragents/cli/commands/audit.py b/src/cleveragents/cli/commands/audit.py new file mode 100644 index 000000000..c34cf814e --- /dev/null +++ b/src/cleveragents/cli/commands/audit.py @@ -0,0 +1,181 @@ +"""CLI commands for audit logging (SEC7). + +Provides ``agents audit`` with subcommands for listing, viewing, +and pruning audit log entries. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +import typer + +from cleveragents.cli.main import get_console, get_err_console + +if TYPE_CHECKING: + from cleveragents.application.services.audit_service import AuditService + +app = typer.Typer( + help="View and manage the audit log for security-relevant operations.", +) + + +def _get_audit_service() -> AuditService: + """Build an :class:`AuditService` from current settings.""" + from cleveragents.application.services.audit_service import AuditService + from cleveragents.config.settings import get_settings + + return AuditService(settings=get_settings()) + + +@app.command(name="list") +def list_entries( + plan: Annotated[ + str | None, + typer.Option("--plan", help="Filter by plan ID."), + ] = None, + project: Annotated[ + str | None, + typer.Option("--project", help="Filter by project name."), + ] = None, + event_type: Annotated[ + str | None, + typer.Option("--type", help="Filter by event type."), + ] = None, + since: Annotated[ + str | None, + typer.Option( + "--since", + help="Only entries after this ISO-8601 timestamp.", + ), + ] = None, + limit: Annotated[ + int, + typer.Option("--limit", "-n", help="Maximum entries to display."), + ] = 50, +) -> None: + """List audit log entries with optional filters.""" + console = get_console() + service = _get_audit_service() + + entries = service.list_entries( + plan_id=plan, + project_name=project, + event_type=event_type, + since=since, + limit=limit, + ) + + if not entries: + console.print("[dim]No audit entries found.[/dim]") + return + + console.print(f"\n[bold]Audit Log ({len(entries)} entries)[/bold]\n") + for entry in entries: + _print_entry_summary(entry) + console.print() + + +@app.command(name="show") +def show_entry( + audit_id: Annotated[ + int, + typer.Argument(help="The audit entry ID to display."), + ], +) -> None: + """Show a single audit entry with full metadata.""" + console = get_console() + err_console = get_err_console() + service = _get_audit_service() + + entry = service.get_entry(audit_id) + if entry is None: + err_console.print(f"[red]Audit entry {audit_id} not found.[/red]") + raise typer.Exit(code=1) + + console.print(f"\n[bold]Audit Entry #{entry.id}[/bold]\n") + console.print(f" Event type: {entry.event_type}") + console.print(f" Created at: {entry.created_at}") + console.print(f" Plan ID: {entry.plan_id or '-'}") + console.print(f" Project: {entry.project_name or '-'}") + console.print(f" Actor: {entry.actor_name or '-'}") + console.print(f" User: {entry.user_identity or '-'}") + console.print(" Details:") + if entry.details: + import json + + for line in json.dumps(entry.details, indent=2).splitlines(): + console.print(f" {line}") + else: + console.print(" (none)") + console.print() + + +@app.command(name="prune") +def prune( + days: Annotated[ + int | None, + typer.Option( + "--days", + help="Delete entries older than this many days. Overrides config.", + ), + ] = None, + yes: Annotated[ + bool, + typer.Option("--yes", "-y", help="Skip confirmation prompt."), + ] = False, +) -> None: + """Delete audit entries older than the retention period.""" + console = get_console() + err_console = get_err_console() + service = _get_audit_service() + + if not yes: + from cleveragents.config.settings import get_settings + + effective_days = ( + days if days is not None else get_settings().audit_retention_days + ) + if effective_days <= 0: + console.print( + "[dim]Retention is set to 0 (keep everything). Nothing to prune.[/dim]" + ) + return + confirm = typer.confirm( + f"Delete audit entries older than {effective_days} days?" + ) + if not confirm: + err_console.print("[yellow]Aborted.[/yellow]") + raise typer.Abort() + + deleted = service.prune(retention_days=days) + console.print(f"[green]Pruned {deleted} audit log entries.[/green]") + + +@app.command(name="count") +def count_entries() -> None: + """Show the total number of audit log entries.""" + console = get_console() + service = _get_audit_service() + total = service.count() + console.print(f"Total audit log entries: {total}") + + +# ── Helpers ─────────────────────────────────────────────────────── + + +def _print_entry_summary(entry: object) -> None: + """Print a one-line summary of an audit entry.""" + from cleveragents.application.services.audit_service import AuditLogEntry + + if not isinstance(entry, AuditLogEntry): + return + console = get_console() + plan_info = f" plan={entry.plan_id}" if entry.plan_id else "" + project_info = f" project={entry.project_name}" if entry.project_name else "" + console.print( + f" [dim]#{entry.id}[/dim] " + f"[bold]{entry.event_type}[/bold]" + f"{plan_info}{project_info} " + f"[dim]{entry.created_at}[/dim]" + ) diff --git a/src/cleveragents/cli/main.py b/src/cleveragents/cli/main.py index f461d8d8e..9dad5b279 100644 --- a/src/cleveragents/cli/main.py +++ b/src/cleveragents/cli/main.py @@ -79,6 +79,7 @@ def _register_subcommands() -> None: from cleveragents.cli.commands import ( action, actor, + audit, cleanup, context, plan, @@ -134,6 +135,11 @@ def _register_subcommands() -> None: name="cleanup", help="Garbage collection and cleanup for stale resources", ) + app.add_typer( + audit.app, + name="audit", + help="View and manage the audit log for security-relevant operations", + ) _subcommands_registered = True diff --git a/src/cleveragents/config/settings.py b/src/cleveragents/config/settings.py index bc3446fe1..829238a6f 100644 --- a/src/cleveragents/config/settings.py +++ b/src/cleveragents/config/settings.py @@ -188,6 +188,17 @@ class Settings(BaseSettings): description="Cleanup schedule: 'manual' (MVP default) or 'auto'.", ) + # Audit logging retention (SEC7) + audit_retention_days: int = Field( + default=0, + ge=0, + validation_alias=AliasChoices("CLEVERAGENTS_AUDIT_RETENTION_DAYS"), + description=( + "Days to retain audit log entries before pruning. " + "0 means keep indefinitely (spec default for compliance)." + ), + ) + # Persistence database_url: str = Field( default="sqlite:///cleveragents.db", diff --git a/src/cleveragents/infrastructure/database/models.py b/src/cleveragents/infrastructure/database/models.py index c5e327c9f..32a1e01b9 100644 --- a/src/cleveragents/infrastructure/database/models.py +++ b/src/cleveragents/infrastructure/database/models.py @@ -2067,6 +2067,52 @@ class AutomationProfileModel(Base): # type: ignore[misc] __table_args__ = (Index("ix_automation_profiles_name", "name"),) +class AuditLogModel(Base): # type: ignore[misc] + """Database model for audit log entries (SEC7). + + Records security-relevant operations: apply events, cancellations, + resource modifications, corrections, config changes, and entity + deletions. Used for compliance, post-hoc analysis, and the + ``agents audit`` CLI commands. + + Table: ``audit_log`` + """ + + __allow_unmapped__ = True + __tablename__ = "audit_log" + + id = Column(Integer, primary_key=True, autoincrement=True) + event_type = Column( + Text, + nullable=False, + doc=( + "Event category: plan_applied, plan_cancelled, " + "resource_modified, correction_applied, config_changed, " + "entity_deleted, session_created" + ), + ) + plan_id = Column(Text, nullable=True) + project_name = Column(Text, nullable=True) + actor_name = Column(Text, nullable=True) + user_identity = Column(Text, nullable=True) + details = Column(Text, nullable=False, doc="JSON-encoded event-specific payload") + created_at = Column( + Text, + nullable=False, + doc="ISO-8601 timestamp, auto-populated on insert", + ) + + __table_args__ = ( + Index("idx_audit_event", "event_type"), + Index( + "idx_audit_plan", + "plan_id", + sqlite_where=Column("plan_id").isnot(None), + ), + Index("idx_audit_created", "created_at"), + ) + + # Database initialization functions def init_database(database_url: str = "sqlite:///.cleveragents/db.sqlite") -> Any: """Initialize the database.