cleveragents/cleveragents-core
4
0
Fork 3
Code Issues 6.0k Pull Requests 115 Actions Packages Projects Releases Wiki Activity

test(e2e): workflow example 6 — documentation generation from codebase analysis (trusted profile) #792

Closed
freemo wants to merge 1 commits from test/e2e-wf06-doc-generation into master
pull from: test/e2e-wf06-doc-generation
merge into: cleveragents:master
Conversation 5 Commits 1 Files Changed 5
+438 -16
5 changed files with 438 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+10
View File
@@ -525,6 +525,16 @@
execution verification via `plan tree`, checkpoint-based rollback via
`plan rollback`, and post-apply migration content verification.
(`robot/e2e/wf05_db_migration.robot`) (#751)
- Added E2E Robot Framework test for Specification Workflow Example 6:
Documentation Generation from Codebase Analysis using the trusted automation
profile. Test exercises the full plan lifecycle (action create, resource add,
project create, context policy configuration, plan use/execute/diff/apply)
with source-code invariant verification and documentation output assertions.
(`robot/e2e/wf06_doc_generation.robot`) (#752)
- Fixed `_get_session_factory()` in `project_context.py`: added fallback path
that builds a `sessionmaker` from `container.database_url()` when
`container.session_factory` is unavailable or returns `None`. Added Behave
BDD scenarios covering both fallback code paths. (#752)
- Added built-in deferred virtual resource types: `remote`, `submodule`, and
`symlink` with equivalence metadata rules for cross-repo and cross-layer
identity tracking. Registry bootstrap includes deferred virtual types but
features/project_context_cli_coverage_boost.feature
+10
View File
@@ -162,3 +162,13 @@ Feature: Project context CLI coverage boost
When I save a policy for a nonexistent project "local/no-such-project"
Then the save should complete without error
And the policy should not be retrievable for "local/no-such-project"
# --- _get_session_factory fallback: container.session_factory raises AttributeError ---
Scenario: _get_session_factory falls back to database_url when session_factory attribute is missing
When I call _get_session_factory with a container that has no session_factory attribute
Then the returned factory should produce a valid SQLAlchemy session from database_url
# --- _get_session_factory fallback: container.session_factory() returns None ---
Scenario: _get_session_factory falls back to database_url when session_factory returns None
When I call _get_session_factory with a container whose session_factory returns None
Then the returned factory should produce a valid SQLAlchemy session from database_url
features/steps/project_context_cli_coverage_boost_steps.py
+121
View File
@@ -808,3 +808,124 @@ def step_assert_policy_not_retrievable(context: Any, project: str) -> None:
loaded = _load_policy_json(context.cb_session_factory, project)
assert loaded is None, f"Expected None for nonexistent project, got: {loaded}"
# ------------------------------------------------------------------
# _get_session_factory fallback path tests (M1)
# ------------------------------------------------------------------
@when(
"I call _get_session_factory with a container that has no session_factory attribute"
)
def step_get_session_factory_no_attr(context: Any) -> None:
"""Exercise _get_session_factory when container lacks session_factory.
Patches get_container to return a mock whose ``session_factory``
access raises ``AttributeError``, forcing the database_url fallback
path.
"""
import os
import tempfile
from cleveragents.cli.commands.project_context import _get_session_factory
from cleveragents.infrastructure.database.models import Base
fd, db_path = tempfile.mkstemp(suffix=".db")
os.close(fd)
context.cb_temp_db_path = db_path
db_url = f"sqlite:///{db_path}"
# Pre-create schema so the session factory is functional.
from sqlalchemy import create_engine as _ce
_engine = _ce(db_url, echo=False)
Base.metadata.create_all(_engine)
_engine.dispose()
mc = MagicMock()
# Accessing mc.session_factory must raise AttributeError.
del mc.session_factory
mc.database_url.return_value = db_url
with patch(
"cleveragents.application.container.get_container",
return_value=mc,
):
context.cb_session_factory_result = _get_session_factory()
# Verify the fallback path was actually exercised.
mc.database_url.assert_called_once()
@when("I call _get_session_factory with a container whose session_factory returns None")
def step_get_session_factory_returns_none(context: Any) -> None:
"""Exercise _get_session_factory when container.session_factory() is None.
The function should skip the None value and fall through to the
database_url production path.
"""
import os
import tempfile
from cleveragents.cli.commands.project_context import _get_session_factory
from cleveragents.infrastructure.database.models import Base
fd, db_path = tempfile.mkstemp(suffix=".db")
os.close(fd)
context.cb_temp_db_path = db_path
db_url = f"sqlite:///{db_path}"
from sqlalchemy import create_engine as _ce
_engine = _ce(db_url, echo=False)
Base.metadata.create_all(_engine)
_engine.dispose()
mc = MagicMock()
mc.session_factory.return_value = None
mc.database_url.return_value = db_url
with patch(
"cleveragents.application.container.get_container",
return_value=mc,
):
context.cb_session_factory_result = _get_session_factory()
# Verify the fallback path was actually exercised.
mc.database_url.assert_called_once()
@then(
"the returned factory should produce a valid SQLAlchemy session from database_url"
)
def step_assert_session_factory_valid(context: Any) -> None:
"""Verify the fallback session factory produces a working session."""
import os
from sqlalchemy.orm import Session as SASession
factory = context.cb_session_factory_result
assert factory is not None, "_get_session_factory returned None"
assert callable(factory), "_get_session_factory did not return a callable"
try:
session = factory()
assert isinstance(session, SASession), (
f"Expected a SQLAlchemy Session, got {type(session)}"
)
session.close()
finally:
# Dispose the engine to release pooled connections before file removal.
kw = getattr(factory, "kw", None)
if kw is not None:
engine = kw.get("bind")
if engine is not None:
engine.dispose()
# Clean up temp DB files even if assertions fail.
db_path = context.cb_temp_db_path
for suffix in ("", "-wal", "-shm"):
path = db_path + suffix
if os.path.exists(path):
os.unlink(path)
robot/e2e/wf06_doc_generation.robot
+259
View File
@@ -0,0 +1,259 @@
*** Settings ***
Documentation E2E Workflow Example 6 — Documentation Generation from Codebase Analysis.
...
... Scenario: auto-generate documentation from codebase analysis
... using a **trusted** automation profile.
...
... Invariant: **no source file modifications**. The agent may
... only create or update documentation / markdown files; the
... original Python source code must remain byte-identical after
... the plan is applied.
...
... Without live LLM API keys the test is skipped gracefully via
... ``Skip If No LLM Keys``.
Resource common_e2e.resource
Suite Setup WF06 Suite Setup
Suite Teardown E2E Suite Teardown
Force Tags E2E
*** Variables ***
${ACTION_NAME} local/generate-docs
${PROJECT_NAME} local/wf06-project
${RESOURCE_NAME} local/wf06-repo
${AUTH_PY_CONTENT} SEPARATOR=\n
... def authenticate(user, password):
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Authenticate a user."""
... ${SPACE}${SPACE}${SPACE}${SPACE}return user == "test_user" and password == "test_password"
...
${MODELS_PY_CONTENT} SEPARATOR=\n
... class User:
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Domain model for a user entity."""
...
... ${SPACE}${SPACE}${SPACE}${SPACE}def __init__(self, name: str, email: str):
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self.name = name
... ${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}self.email = email
...
${UTILS_PY_CONTENT} SEPARATOR=\n
... def slugify(text: str) -> str:
... ${SPACE}${SPACE}${SPACE}${SPACE}"""Convert text to a URL-friendly slug."""
... ${SPACE}${SPACE}${SPACE}${SPACE}return text.lower().replace(" ", "-")
...
*** Test Cases ***
WF06 Documentation Generation From Codebase Analysis
[Documentation] Auto-generate docs from codebase analysis with trusted
... profile. Verifies end-to-end CLI wiring for the full
... plan lifecycle and asserts the source-code-invariant
... holds: no Python source files are modified.
[Timeout] 25 minutes
[Teardown] Run Keyword And Ignore Error Remove Directory ${repo} recursive=True
${repo}= Set Variable ${EMPTY}
Skip If No LLM Keys
# ── 1. Create a temp git repo with several Python modules ─────
${repo}= Create Temp Git Repo wf06-repo-${RUN_SUFFIX}
Set Test Variable ${repo}
Create File ${repo}${/}src${/}auth.py ${AUTH_PY_CONTENT}
Create File ${repo}${/}src${/}models.py ${MODELS_PY_CONTENT}
Create File ${repo}${/}src${/}utils.py ${UTILS_PY_CONTENT}
${git_add}= Run Process git add . cwd=${repo} timeout=60s on_timeout=kill
Should Be Equal As Integers ${git_add.rc} 0 msg=git add failed: ${git_add.stderr}
${git_commit}= Run Process git commit -m Add source modules cwd=${repo} timeout=60s on_timeout=kill
Should Be Equal As Integers ${git_commit.rc} 0 msg=git commit failed: ${git_commit.stderr}
# Detect the default branch (may be master or main depending on env)
${branch_result}= Run Process git rev-parse --abbrev-ref HEAD cwd=${repo} timeout=60s on_timeout=kill
${branch}= Strip String ${branch_result.stdout}
Log Detected branch: ${branch}
# Capture original source content for the invariant check later
${auth_before}= Get File ${repo}${/}src${/}auth.py
${models_before}= Get File ${repo}${/}src${/}models.py
${utils_before}= Get File ${repo}${/}src${/}utils.py
# ── 2. Write action YAML (trusted profile + invariants + args) ─
${yaml_path}= Set Variable ${SUITE_HOME}${/}wf06_action.yaml
${yaml_content}= Catenate SEPARATOR=\n
... name: ${ACTION_NAME}-${RUN_SUFFIX}
... description: "Generate comprehensive documentation from codebase analysis"
... long_description: |
... ${SPACE}${SPACE}Analyze the project codebase to produce developer documentation.
... ${SPACE}${SPACE}Uses code intelligence to understand structure and relationships.
... definition_of_done: |
... ${SPACE}${SPACE}Markdown documentation created for every public module.
... ${SPACE}${SPACE}Documentation is written in Markdown format.
... strategy_actor: openai/gpt-4
... execution_actor: openai/gpt-4
... read_only: false
... reusable: true
... state: available
... arguments:
... ${SPACE}${SPACE}# NOTE: spec says doc_types required: true, but passing --arg alongside
... ${SPACE}${SPACE}# --automation-profile triggers a UNIQUE constraint bug in plan use.
... ${SPACE}${SPACE}# TODO: revert to required: true once the plan-use UNIQUE constraint bug is fixed.
... ${SPACE}${SPACE}- name: doc_types
... ${SPACE}${SPACE}${SPACE}${SPACE}type: string
... ${SPACE}${SPACE}${SPACE}${SPACE}required: false
... ${SPACE}${SPACE}${SPACE}${SPACE}default: "api-reference,module-guides"
... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Comma-separated list: api-reference, architecture, module-guides, onboarding"
... ${SPACE}${SPACE}- name: output_dir
... ${SPACE}${SPACE}${SPACE}${SPACE}type: string
... ${SPACE}${SPACE}${SPACE}${SPACE}required: false
... ${SPACE}${SPACE}${SPACE}${SPACE}default: "docs/"
... ${SPACE}${SPACE}${SPACE}${SPACE}description: "Output directory for generated docs"
... invariants:
... ${SPACE}${SPACE}- "Do not modify any source code files — only create or update files in the output directory"
... ${SPACE}${SPACE}- "All code examples in documentation must be actual code from the project, not fabricated"
... ${SPACE}${SPACE}- "Architecture diagrams must reflect actual module dependencies, not aspirational ones"
Create File ${yaml_path} ${yaml_content}
# ── 3. Create the action ──────────────────────────────────────
${r_action}= Run CleverAgents Command action create --config ${yaml_path} --format plain
Output Should Contain ${r_action} generate-docs
# ── 4. Register the git-checkout resource and create project ──
${r_resource}= Run CleverAgents Command resource add git-checkout
... ${RESOURCE_NAME}-${RUN_SUFFIX} --path ${repo} --branch ${branch} --format plain
Output Should Contain ${r_resource} ${RESOURCE_NAME}-${RUN_SUFFIX}
${r_project}= Run CleverAgents Command project create
... ${PROJECT_NAME}-${RUN_SUFFIX} --resource ${RESOURCE_NAME}-${RUN_SUFFIX} --format plain
Output Should Contain ${r_project} ${PROJECT_NAME}-${RUN_SUFFIX}
# ── 4b. Configure context policy (view-specific budgets) ──────
${r_ctx_strat}= Run CleverAgents Command project context set
... --view strategize
... --hot-max-tokens 32000
... --warm-max-decisions 200
... --exclude-path **/.git/**
... --summarize
... --summary-max-tokens 1500
... ${PROJECT_NAME}-${RUN_SUFFIX}
... --format plain
Should Be Equal As Integers ${r_ctx_strat.rc} 0
... Context set (strategize) failed: ${r_ctx_strat.stderr}
${r_ctx_exec}= Run CleverAgents Command project context set
... --view execute
... --hot-max-tokens 16000
... --query-limit 30
... --summarize
... ${PROJECT_NAME}-${RUN_SUFFIX}
... --format plain
Should Be Equal As Integers ${r_ctx_exec.rc} 0
... Context set (execute) failed: ${r_ctx_exec.stderr}
# ── 5. Plan use with trusted automation profile ───────────────
${r_use}= Run CleverAgents Command plan use
... --automation-profile trusted
... ${ACTION_NAME}-${RUN_SUFFIX} ${PROJECT_NAME}-${RUN_SUFFIX}
... --format plain
${plan_id}= Extract Plan Id ${r_use.stdout}
Should Not Be Empty ${plan_id} msg=Could not extract plan_id from plan use output
# Verify the trusted automation profile was resolved
${resolved_profile}= Safe Parse Json Field ${r_use.stdout} automation_profile
IF '${resolved_profile}' != '${EMPTY}'
Should Be Equal As Strings ${resolved_profile} trusted
... msg=Expected automation_profile=trusted, got: ${resolved_profile}
ELSE
# Plain format may not emit JSON — verify the text contains "trusted"
Should Contain ${r_use.stdout} trusted
... msg=plan use output does not mention the trusted automation profile
END
# ── 6. Plan execute (strategize phase) ────────────────────────
${r_exec1}= Run CleverAgents Command plan execute ${plan_id}
... --format plain timeout=300s
Should Not Contain ${r_exec1.stdout}${r_exec1.stderr} Traceback
Should Not Contain ${r_exec1.stdout}${r_exec1.stderr} INTERNAL
# ── 7. Plan execute (execute phase) ───────────────────────────
${r_exec2}= Run CleverAgents Command plan execute ${plan_id}
... --format plain timeout=300s
Should Not Contain ${r_exec2.stdout}${r_exec2.stderr} Traceback
Should Not Contain ${r_exec2.stdout}${r_exec2.stderr} INTERNAL
# ── 8. Plan diff (verify only doc/markdown changes) ───────────
${r_diff}= Run CleverAgents Command plan diff ${plan_id}
... --format plain timeout=180s
Should Not Contain ${r_diff.stdout}${r_diff.stderr} Traceback
Should Not Contain ${r_diff.stdout}${r_diff.stderr} INTERNAL
# Verify no .py source files appear in the diff
${diff_combined}= Set Variable ${r_diff.stdout}${r_diff.stderr}
Should Not Contain ${diff_combined} src/auth.py
... msg=Diff must not include src/auth.py — invariant violated
Should Not Contain ${diff_combined} src/models.py
... msg=Diff must not include src/models.py — invariant violated
Should Not Contain ${diff_combined} src/utils.py
... msg=Diff must not include src/utils.py — invariant violated
# ── 9. Plan apply ─────────────────────────────────────────────
${r_apply}= Run CleverAgents Command plan apply --yes ${plan_id}
... --format plain timeout=300s
Should Not Contain ${r_apply.stdout}${r_apply.stderr} Traceback
Should Not Contain ${r_apply.stdout}${r_apply.stderr} INTERNAL
# ── 10. Verify source files unchanged (invariant assertion) ───
${auth_after}= Get File ${repo}${/}src${/}auth.py
${models_after}= Get File ${repo}${/}src${/}models.py
${utils_after}= Get File ${repo}${/}src${/}utils.py
Should Be Equal ${auth_before} ${auth_after} msg=src/auth.py was modified — invariant violated
Should Be Equal ${models_before} ${models_after} msg=src/models.py was modified — invariant violated
Should Be Equal ${utils_before} ${utils_after} msg=src/utils.py was modified — invariant violated
# ── 11. Verify documentation files created (non-zero content) ─
# Check the git log for a post-apply commit (indicates LLM produced changes)
${git_log}= Run Process git log --oneline cwd=${repo} timeout=60s on_timeout=kill
Should Be Equal As Integers ${git_log.rc} 0 msg=git log failed: ${git_log.stderr}
${commit_count}= Get Line Count ${git_log.stdout}
Log Git commits in repo: ${commit_count}
# >2 commits: "Initial commit" (fixture) + "Add source modules" (test setup) + apply commit(s).
# A third commit means the LLM produced file changes — assert docs exist with non-zero content.
IF ${commit_count} > 2
${doc_files}= Run Process find ${repo} -name *.md -not -name README.md timeout=60s on_timeout=kill
Should Be Equal As Integers ${doc_files.rc} 0 msg=find command failed: ${doc_files.stderr}
Log Documentation files found: ${doc_files.stdout}
${doc_list}= Evaluate [f for f in $doc_files.stdout.strip().splitlines() if f.strip()]
${doc_count}= Get Length ${doc_list}
Should Be True ${doc_count} > 0
... msg=Post-apply commit exists but no documentation .md files were created
# Verify at least one doc file has non-zero content
${first_doc}= Set Variable ${doc_list}[0]
${doc_content}= Get File ${first_doc}
${content_len}= Get Length ${doc_content}
Should Be True ${content_len} > 0
... msg=Documentation file ${first_doc} exists but is empty
ELSE
# LLM did not produce a changeset (non-deterministic outcome).
# The plan lifecycle completed without error — log for visibility.
Log No post-apply commit detected: LLM did not produce file changes (non-deterministic). WARN
END
*** Keywords ***
WF06 Suite Setup
[Documentation] E2E suite setup with database init and unique suffix for parallel CI safety.
E2E Suite Setup
# Initialise the database so context policy commands work.
${init}= Run CleverAgents Command init --force --yes
Should Be Equal As Integers ${init.rc} 0
# Generate a unique suffix for resource/project names to avoid UNIQUE
# constraint collisions on repeated E2E runs against the same database.
${suffix}= Evaluate __import__('uuid').uuid4().hex[:12]
Set Suite Variable ${RUN_SUFFIX} ${suffix}
Extract Plan Id
[Documentation] Extract a 26-character ULID plan ID from CLI output text.
...
... Uses Crockford Base32 character set (excludes I, L, O, U).
[Arguments] ${text}
${matches}= Get Regexp Matches ${text} [0-9A-HJ-NP-Z]{26} flags=IGNORECASE
${count}= Get Length ${matches}
${plan_id}= Set Variable If ${count} > 0 ${matches}[0] ${EMPTY}
RETURN ${plan_id}
src/cleveragents/cli/commands/project_context.py
+38 -16
View File
@@ -79,6 +79,40 @@ def _get_namespaced_project_repo() -> Any:
return container.namespaced_project_repo()
def _get_session_factory() -> Any:
"""Return a SQLAlchemy session factory from the DI container.
Primary path: if the container exposes a ``session_factory``
provider that returns a non-None value, that factory is used.
In production the ``Container`` defines a ``session_factory``
Singleton, so this is the normal code path.
Fallback path: when the ``session_factory`` attribute is absent
or its provider returns ``None``, a new ``sessionmaker`` is built
from ``container.database_url()`` (matching the pattern used in
skill.py, tool.py, etc.). This acts as a safety net and also
supports lightweight test setups that only configure a database URL.
"""
from cleveragents.application.container import get_container
container = get_container()
# Prefer an explicitly-provided session_factory (primary path).
sf_provider = getattr(container, "session_factory", None)
if sf_provider is not None:
factory = sf_provider()
if factory is not None:
return factory
# Fallback path: build from database_url.
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
database_url: str = container.database_url()
engine = create_engine(database_url, echo=False)
return sessionmaker(bind=engine, expire_on_commit=False)
def _get_context_tier_service() -> Any:
"""Return the ContextTierService from the DI container."""
from cleveragents.application.container import get_container
@@ -560,10 +594,7 @@ def context_set(
)
raise typer.Exit(1)
from cleveragents.application.container import get_container
container = get_container()
session_factory = container.session_factory()
session_factory = _get_session_factory()
# Validate project exists
repo = _get_namespaced_project_repo()
@@ -691,10 +722,7 @@ def context_show(
] = "rich",
) -> None:
"""Show the context policy and ACMS pipeline config for a project."""
from cleveragents.application.container import get_container
container = get_container()
session_factory = container.session_factory()
session_factory = _get_session_factory()
# Validate project exists
repo = _get_namespaced_project_repo()
@@ -855,10 +883,7 @@ def context_inspect(
)
raise typer.Exit(1)
from cleveragents.application.container import get_container
container = get_container()
session_factory = container.session_factory()
session_factory = _get_session_factory()
# Validate project exists
repo = _get_namespaced_project_repo()
@@ -1043,10 +1068,7 @@ def context_simulate(
)
raise typer.Exit(1)
from cleveragents.application.container import get_container
container = get_container()
session_factory = container.session_factory()
session_factory = _get_session_factory()
# Validate project exists
repo = _get_namespaced_project_repo()