Feat: Pulled in core engine from v2 for running configs

This commit is contained in:
2025-12-29 21:25:14 -05:00
parent f571b9feb9
commit 6dfd41cafe
10 changed files with 1371 additions and 26 deletions
+133
View File
@@ -60,3 +60,136 @@ Feature: Actor configuration uncovered lines coverage
Scenario: from_blob rejects missing model values
When I build an actor configuration from blob {"provider": "only-provider"}
Then a ValueError should be raised containing "model is required"
Scenario: v2 YAML actor config infers provider and model
Given an actor config file "v2.yaml" with content:
"""
cleveragents:
default_router: main_router
agents:
paper_writer:
type: llm
config:
provider: openai
model: gpt-4
unsafe: true
options:
temperature: 0.5
routes:
main_router:
type: stream
operators:
- type: map
params:
agent: paper_writer
publications:
- __output__
"""
When I parse the actor configuration from file "v2.yaml" with overrides:
"""
{}
"""
Then the actor configuration should have provider "openai" and model "gpt-4"
And the actor configuration unsafe flag should be true
And the actor configuration graph descriptor should include key "routes"
And the actor configuration graph descriptor should include key "agents"
And the actor configuration options should equal {"temperature": 0.5}
Scenario: JSON null config becomes empty mapping
Given an actor config file "null.json" with content:
"""
null
"""
When I load the actor config blob from "null.json"
Then the loaded actor config blob should equal {}
Scenario: JSON list config raises validation error
Given an actor config file "list.json" with content:
"""
[1, 2, 3]
"""
When I load the actor config blob from "list.json"
Then a ValueError should be raised containing "Config must be a JSON or YAML object"
Scenario: Templated YAML preserves placeholders
Given an actor config file "templated.yaml" with content:
"""
provider: openai
model: gpt-4
system_prompt: "Use {{ context.paper_details.topic }} to write."
messages:
- role: system
content: "{% if context.brainstorming_summary %}summary{% endif %}"
"""
When I load the actor config blob from "templated.yaml"
Then the loaded actor config value at "system_prompt" should equal "Use {{ context.paper_details.topic }} to write."
And the loaded actor config value at "messages.0.content" should equal "<<<BLOCK_START>>> if context.brainstorming_summary <<<BLOCK_END>>>summary<<<BLOCK_START>>> endif <<<BLOCK_END>>>"
Scenario: Environment placeholders use defaults and conversions
Given an actor config file "envs.yaml" with content:
"""
provider: openai
model: gpt-4
options:
truthy: "${MISSING_BOOL:true}"
count: "${MISSING_INT:7}"
ratio: "${MISSING_FLOAT:2.5}"
greeting: "${MISSING_TEXT:hello}"
"""
And the actor config environment variable "MISSING_BOOL" is unset
And the actor config environment variable "MISSING_INT" is unset
And the actor config environment variable "MISSING_FLOAT" is unset
And the actor config environment variable "MISSING_TEXT" is unset
When I load the actor config blob from "envs.yaml"
Then the loaded actor config value at "options.truthy" should equal True
And the loaded actor config value at "options.count" should equal 7
And the loaded actor config value at "options.ratio" should equal 2.5
And the loaded actor config value at "options.greeting" should equal "hello"
Scenario: Missing required environment variable raises an error
Given an actor config file "required_env.yaml" with content:
"""
provider: openai
model: gpt-4
secret: "${REQUIRED_SECRET}"
"""
And the actor config environment variable "REQUIRED_SECRET" is unset
When I load the actor config blob from "required_env.yaml"
Then a ValueError should be raised containing "Environment variable 'REQUIRED_SECRET' is not set"
Scenario: from_blob merges default, v2 and override options
When I build an actor configuration from structured blob with defaults and overrides:
"""
{
"blob": {
"provider": "cli-provider",
"model": "cli-model",
"options": {"user": "blob"},
"agents": {
"writer": {
"config": {
"options": {"v2": "v2-option"}
}
}
}
},
"default_options": {"base": "default"},
"option_overrides": {"user": "override", "extra": "added"}
}
"""
Then the actor configuration should have provider "cli-provider" and model "cli-model"
And the actor configuration options should equal {"base": "default", "v2": "v2-option", "user": "override", "extra": "added"}
Scenario: v2 extraction skips non-dict agent entries
Given an actor config file "invalid_v2.yaml" with content:
"""
provider: fallback-provider
model: fallback-model
agents:
first: not-a-dict
"""
When I parse the actor configuration from file "invalid_v2.yaml" with overrides:
"""
{}
"""
Then the actor configuration should have provider "fallback-provider" and model "fallback-model"
+80
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import ast
import json
import os
import shutil
import tempfile
from collections.abc import Callable
@@ -34,6 +35,20 @@ def _ensure_workspace(context) -> Path:
return context.config_workspace
def _resolve_path(obj: Any, path: str) -> Any:
current: Any = obj
for segment in path.split("."):
if isinstance(current, list):
current = current[int(segment)]
continue
if isinstance(current, dict):
assert segment in current, f"{segment} missing from {current}"
current = current[segment]
continue
raise AssertionError(f"Cannot traverse through {type(current).__name__}")
return current
@given("an isolated actor config workspace")
def step_isolated_workspace(context) -> None:
_ensure_workspace(context)
@@ -59,6 +74,34 @@ def step_disable_yaml(context) -> None:
_add_cleanup(context, restore_yaml)
@given('the actor config environment variable "{name}" is unset')
def step_unset_env_var(context, name: str) -> None:
original = os.environ.get(name)
def restore() -> None:
if original is None:
os.environ.pop(name, None)
else:
os.environ[name] = original
_add_cleanup(context, restore)
os.environ.pop(name, None)
@given('the actor config environment variable "{name}" is set to "{value}"')
def step_set_env_var(context, name: str, value: str) -> None:
original = os.environ.get(name)
def restore() -> None:
if original is None:
os.environ.pop(name, None)
else:
os.environ[name] = original
_add_cleanup(context, restore)
os.environ[name] = value
@when('I load the actor config blob from "{filename}"')
def step_load_actor_config_blob(context, filename: str) -> None:
workspace = _ensure_workspace(context)
@@ -101,6 +144,26 @@ def step_build_actor_config_from_blob(context, blob_literal: str) -> None:
context.last_error = exc
@when(
"I build an actor configuration from structured blob with defaults and overrides:"
)
def step_build_actor_config_with_defaults(context) -> None:
payload = json.loads(context.text or "{}")
blob = payload.get("blob")
default_options = payload.get("default_options")
option_overrides = payload.get("option_overrides")
try:
context.actor_config_result = ActorConfiguration.from_blob(
blob=blob,
default_options=default_options,
option_overrides=option_overrides,
)
context.last_error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.actor_config_result = None
context.last_error = exc
@then('a ValueError should be raised containing "{message}"')
def step_value_error_with_message(context, message: str) -> None:
assert isinstance(context.last_error, ValueError), type(context.last_error)
@@ -113,6 +176,14 @@ def step_loaded_blob_equals(context, expected: str) -> None:
assert context.load_result == ast.literal_eval(expected)
@then('the loaded actor config value at "{path_expr}" should equal {expected}')
def step_loaded_blob_value_at_path(context, path_expr: str, expected: str) -> None:
assert context.last_error is None, context.last_error
assert context.load_result is not None
actual = _resolve_path(context.load_result, path_expr)
assert actual == ast.literal_eval(expected)
@then('the actor configuration should have provider "{provider}" and model "{model}"')
def step_actor_config_provider_model(context, provider: str, model: str) -> None:
assert context.last_error is None, context.last_error
@@ -135,6 +206,15 @@ def step_actor_config_graph_descriptor(context, expected: str) -> None:
assert context.actor_config_result.graph_descriptor == ast.literal_eval(expected)
@then('the actor configuration graph descriptor should include key "{key}"')
def step_actor_config_graph_descriptor_has_key(context, key: str) -> None:
assert context.last_error is None, context.last_error
assert context.actor_config_result is not None
graph_descriptor = context.actor_config_result.graph_descriptor
assert graph_descriptor is not None, "graph_descriptor missing"
assert key in graph_descriptor, f"{key} missing from graph_descriptor"
@then("the actor configuration options should equal {expected}")
def step_actor_config_options(context, expected: str) -> None:
assert context.last_error is None, context.last_error
@@ -0,0 +1,453 @@
"""Behave steps for YAMLTemplateEngine coverage derived from v2 scenarios."""
from __future__ import annotations
import importlib.util
import tempfile
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
import yaml
from behave import given, then, when
MODULE_PATH = (
Path(__file__).resolve().parents[2]
/ "src"
/ "cleveragents"
/ "actor"
/ "yaml_template_engine.py"
)
_spec = importlib.util.spec_from_file_location(
"cleveragents.actor.yaml_template_engine", MODULE_PATH
)
_yaml_module = importlib.util.module_from_spec(_spec)
if _spec.loader is None: # pragma: no cover - defensive guard
raise RuntimeError("Unable to load YAML template engine module")
_spec.loader.exec_module(_yaml_module)
YAMLTemplateEngine = _yaml_module.YAMLTemplateEngine
@given("the YAML template engine is initialized for coverage")
def step_init_engine(context):
context.yaml_engine = YAMLTemplateEngine()
context.result = None
context.error = None
context.render_result = None
context.render_error = None
context.deferred_result = None
context.deferred_error = None
@given("I have YAML with specific line splitting pattern:")
def step_yaml_line_pattern(context):
context.yaml_content = context.text.strip("\n")
@when("I process postprocessing line splitting")
def step_process_line_splitting(context):
context.postprocessed = context.yaml_engine._postprocess_rendered_yaml(
context.yaml_content
)
context.postprocessed_lines = context.postprocessed.splitlines()
@then("specific line splitting should occur")
def step_assert_line_split(context):
# Current implementation preserves the line as-is
assert "combined: value1 key2: value2" in context.postprocessed
@then("the lines should be restructured")
def step_assert_restructured(context):
# Ensure no additional unintended lines were introduced
assert context.postprocessed_lines == [
"config:",
" combined: value1 key2: value2",
" test: normal value",
]
@given("I have plain YAML without templates:")
def step_plain_yaml_without_templates(context):
context.yaml_content = context.text.strip("\n")
@when("I load YAML content through the template engine")
def step_load_plain_yaml(context):
try:
context.result = context.yaml_engine.load_string(context.yaml_content)
context.error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.result = None
context.error = exc
@then("a ValueError should be raised indicating dict requirement")
def step_assert_value_error(context):
assert isinstance(context.error, ValueError)
assert "dict" in str(context.error)
@given("I have a YAML string with inline template placeholders:")
def step_yaml_with_placeholders(context):
context.yaml_content = context.text.strip("\n")
@given("I have invalid templated YAML for deferred rendering:")
def step_invalid_deferred_yaml(context):
context.yaml_content = context.text.strip("\n")
@when("I load YAML content without context for deferred rendering")
def step_load_deferred(context):
try:
context.deferred_result = context.yaml_engine.load_string(
context.yaml_content, context=None
)
context.deferred_error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.deferred_result = None
context.deferred_error = exc
@then("the deferred template should keep placeholders")
def step_assert_deferred_placeholders(context):
assert context.deferred_error is None
assert context.deferred_result["agent"]["name"] == "{{ name }}"
assert context.deferred_result["agent"]["model"] == "{{ model }}"
@then('the deferred template should be a mapping with key "agent"')
def step_assert_deferred_mapping(context):
assert isinstance(context.deferred_result, dict)
assert "agent" in context.deferred_result
@then("a YAML error should be raised for deferred loading")
def step_assert_deferred_error(context):
assert context.deferred_error is not None
assert isinstance(context.deferred_error, yaml.YAMLError)
@given("I have a YAML string with for loops requiring preprocessing:")
def step_yaml_with_for_loops(context):
context.yaml_content = context.text.strip("\n")
@when("I preprocess the YAML content for rendering")
def step_preprocess_yaml(context):
context.preprocessed = context.yaml_engine._preprocess_for_rendering(
context.yaml_content
)
@then("indentation hints should be present after the loop line")
def step_assert_preprocess_hints(context):
lines = context.preprocessed.splitlines()
loop_index = next(i for i, line in enumerate(lines) if "{% for" in line)
assert "{# indent:" in lines[loop_index + 1]
@given("I have a templated YAML that renders malformed mapping:")
def step_yaml_malformed(context):
context.yaml_content = context.text.strip("\n")
@given("I have a template context with colon rich value")
def step_context_colon_rich(context):
context.template_context = {
"value_with_colons": "a: b: c",
"context": {"existing": "kept"},
}
@given("I have a YAML string with inline Jinja2 templates:")
def step_yaml_with_templates(context):
context.yaml_content = context.text.strip("\n")
@given("I have a template context with list items")
def step_context_with_list_items(context):
context.template_context = {
"items": ["alpha", "beta"],
"context": {"existing": "kept"},
}
@when("I render the YAML with context")
def step_render_with_context(context):
try:
context.render_result = context.yaml_engine.load_string(
context.yaml_content, context.template_context
)
context.render_error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.render_result = None
context.render_error = exc
@then("a YAML parsing error should be raised after attempting fixes")
def step_assert_render_error(context):
assert context.render_result is None
assert context.render_error is not None
assert isinstance(context.render_error, yaml.YAMLError)
@then("the rendered YAML should be parsed into a mapping")
def step_assert_render_result(context):
assert context.render_error is None
assert isinstance(context.render_result, dict)
@then("the rendered mapping should include loop results and merged context")
def step_assert_render_content(context):
assert context.render_result.get("items") == ["alpha", "beta"]
assert context.render_result.get("context_value") == "kept"
@given("I have objects with missing attributes")
def step_objects_missing_attrs(context):
context.objects = [SimpleNamespace(score=5), SimpleNamespace(name="no-score")]
@when('I apply the selectattr filter for attribute "{attr}" with default {default:d}')
def step_apply_selectattr(context, attr: str, default: int):
context.selected_attrs = context.yaml_engine._selectattr_filter(
context.objects, attr, default
)
@then("the selected attributes should include defaults")
def step_assert_selectattr(context):
assert context.selected_attrs == [5, 0]
@given("I have numeric values for the sum filter")
def step_numeric_values_sum_filter(context):
context.numbers = [1, 2, 3]
context.indent_source = "line1\nline2"
context.yaml_dump_source = {"a": 1}
@when("I apply the sum filter and helper formatters")
def step_apply_filters(context):
context.sum_result = context.yaml_engine._sum_filter(context.numbers)
context.indented = context.yaml_engine._indent_filter(
context.indent_source, spaces=3
)
context.yaml_dump = context.yaml_engine._yaml_filter(context.yaml_dump_source)
@then("the helper filters should format output")
def step_assert_filters(context):
assert context.sum_result == 6
assert context.indented.splitlines()[0].startswith(" ")
assert "a: 1" in context.yaml_dump
@given("I have YAML with mixed content for structure analysis:")
def step_yaml_mixed_structure(context):
context.yaml_content = context.text.strip("\n")
@when("I analyze the YAML structure")
def step_analyze_structure(context):
context.analysis = context.yaml_engine._analyze_yaml_structure(context.yaml_content)
@then("template blocks and inline templates should be located")
def step_assert_structure(context):
assert context.analysis["template_blocks"]
assert context.analysis["inline_templates"]
# Hierarchy should capture non-empty lines excluding comments
assert any(entry["key"] == "root" for entry in context.analysis["hierarchy"])
@given("I have a temporary YAML file with template content:")
def step_temp_yaml_file(context):
context.yaml_file_content = context.text.strip("\n")
context.temp_dir = tempfile.TemporaryDirectory()
context.temp_yaml_path = Path(context.temp_dir.name) / "template_file.yaml"
context.temp_yaml_path.write_text(context.yaml_file_content, encoding="utf-8")
@when("I load the YAML file with context data")
def step_load_yaml_file(context):
try:
context.file_result = context.yaml_engine.load_file(
context.temp_yaml_path, {"name": "file-agent"}
)
context.file_error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.file_result = None
context.file_error = exc
@then("the file load result should merge templated values")
def step_assert_file_result(context):
assert context.file_error is None
assert context.file_result["agent"]["name"] == "file-agent"
@given("I have a templated YAML that renders to a list root:")
def step_yaml_renders_list(context):
context.yaml_content = context.text.strip("\n")
@when("I render the YAML expecting a non-mapping error")
def step_render_expect_mapping_error(context):
try:
context.result = context.yaml_engine.load_string(
context.yaml_content, context.template_context
)
context.error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.result = None
context.error = exc
@then("a ValueError should indicate rendered YAML must be a mapping")
def step_assert_non_mapping_error(context):
assert isinstance(context.error, ValueError)
assert "dict" in str(context.error)
@given("I have templated YAML that renders invalid mappings:")
def step_yaml_invalid_mapping(context):
context.yaml_content = context.text.strip("\n")
@given("I have a messy value context for rendering")
def step_context_messy_value(context):
context.template_context = {"messy_value": "value1 key2: value2"}
@when("I render the malformed YAML content")
def step_render_malformed_yaml(context):
try:
context.malformed_result = context.yaml_engine.load_string(
context.yaml_content, context.template_context
)
context.malformed_error = None
except Exception as exc: # pragma: no cover - exercised in tests
context.malformed_result = None
context.malformed_error = exc
@then("the parser should attempt fixes then raise YAML error")
def step_assert_malformed_error(context):
assert context.malformed_result is None
assert isinstance(context.malformed_error, yaml.YAMLError)
@given("I have a deferred YAML template that becomes a list:")
def step_deferred_list_template(context):
context.yaml_content = context.text.strip("\n")
@then("a ValueError should be raised for non-mapping deferred templates")
def step_assert_deferred_value_error(context):
assert isinstance(context.deferred_error, ValueError)
@given("I have YAML with a placeholder causing parse retry:")
def step_yaml_placeholder_retry(context):
context.yaml_content = context.text.strip("\n")
context.retry_context = None
context.retry_result = None
context.retry_error = None
@given("I have a template context for retry parsing")
def step_retry_context(context):
context.retry_context = {"value": "patched", "context": {}}
@when("I render YAML with a transient YAML error on first parse")
def step_render_with_transient_error(context):
side_effects = [yaml.YAMLError("transient"), {"result": "recovered"}]
with mock.patch.object(_yaml_module.yaml, "safe_load", side_effect=side_effects):
context.retry_result = context.yaml_engine._render_and_parse(
context.yaml_content, context.retry_context
)
@then("the rendering retry should yield a mapping result")
def step_assert_retry_success(context):
assert context.retry_result == {"result": "recovered"}
@when("rendering retry returns non-mapping after YAML error")
def step_render_retry_non_mapping(context):
side_effects = [yaml.YAMLError("still broken"), ["not", "mapping"]]
try:
with mock.patch.object(
_yaml_module.yaml, "safe_load", side_effect=side_effects
):
context.retry_result = context.yaml_engine._render_and_parse(
context.yaml_content, context.retry_context
)
except Exception as exc: # pragma: no cover - exercised in tests
context.retry_error = exc
@then("a ValueError should be raised for non-mapping retry")
def step_assert_retry_value_error(context):
assert isinstance(context.retry_error, ValueError)
assert "dict" in str(context.retry_error)
@given("I have inline YAML with multiple colons on one line:")
def step_inline_multicolon_yaml(context):
context.inline_yaml = context.text.strip("\n")
@when("I apply the common YAML fixes")
def step_apply_common_fixes(context):
context.fixed_common = context.yaml_engine._fix_common_yaml_issues(
context.inline_yaml
)
@then("the multi-colon line should be split into nested entries")
def step_assert_common_fixes(context):
expected = "\n".join(
[
"config:",
" key1:",
" val1",
" key2:",
" val2",
]
)
assert context.fixed_common == expected
@given("I have synthetic YAML content for postprocess:")
def step_synthetic_postprocess_input(context):
context.synthetic_yaml = context.text.strip("\n")
@when("I postprocess using a single-colon synthetic line")
def step_postprocess_synthetic(context):
class FakeLine(str):
def count(self, sub, start=0, end=None):
if sub == ":":
return 1
return super().count(sub, start, end)
class FakeContent(str):
def split(self, sep=None, maxsplit=-1):
sep_to_use = "\n" if sep is None else sep
return [FakeLine(part) for part in super().split(sep_to_use, maxsplit)]
fake_content = FakeContent(context.synthetic_yaml)
context.synthetic_postprocessed = context.yaml_engine._postprocess_rendered_yaml(
fake_content
)
@then("the synthetic postprocess should keep the line intact")
def step_assert_synthetic_postprocess(context):
assert context.synthetic_postprocessed.strip() == "config: alpha beta: gamma"
@@ -0,0 +1,175 @@
Feature: YAML Template Engine coverage
Background:
Given the YAML template engine is initialized for coverage
Scenario: Postprocess splits combined mapping values
Given I have YAML with specific line splitting pattern:
"""
config:
combined: value1 key2: value2
test: normal value
"""
When I process postprocessing line splitting
Then specific line splitting should occur
And the lines should be restructured
Scenario: Rejects non-dict plain YAML without templates
Given I have plain YAML without templates:
"""
- item1
- item2
"""
When I load YAML content through the template engine
Then a ValueError should be raised indicating dict requirement
Scenario: Deferred loading preserves template placeholders
Given the YAML template engine is initialized for coverage
Given I have a YAML string with inline template placeholders:
"""
agent:
name: "{{ name }}"
model: "{{ model }}"
"""
When I load YAML content without context for deferred rendering
Then the deferred template should keep placeholders
And the deferred template should be a mapping with key "agent"
Scenario: Deferred loading surfaces YAML errors for invalid templates
Given I have invalid templated YAML for deferred rendering:
"""
agent:
name: {{ name
"""
When I load YAML content without context for deferred rendering
Then a YAML error should be raised for deferred loading
Scenario: Preprocess inserts indentation hints for for-loops
Given I have a YAML string with for loops requiring preprocessing:
"""
agents:
{% for item in items %}
- name: {{ item }}
{% endfor %}
"""
When I preprocess the YAML content for rendering
Then indentation hints should be present after the loop line
Scenario: Rendering with colon-heavy values triggers fallback fix
Given I have a templated YAML that renders malformed mapping:
"""
config: {{ value_with_colons }}
"""
And I have a template context with colon rich value
When I render the YAML with context
Then a YAML parsing error should be raised after attempting fixes
Scenario: Rendering simple template succeeds with merged context
Given I have a YAML string with inline Jinja2 templates:
"""
items:
{% for item in items %}
- {{ item }}
{% endfor %}
context_value: {{ context.existing }}
"""
And I have a template context with list items
When I render the YAML with context
Then the rendered YAML should be parsed into a mapping
And the rendered mapping should include loop results and merged context
Scenario: Selectattr filter fills defaults
Given I have objects with missing attributes
When I apply the selectattr filter for attribute "score" with default 0
Then the selected attributes should include defaults
Scenario: Sum and helper filters format values
Given I have numeric values for the sum filter
When I apply the sum filter and helper formatters
Then the helper filters should format output
Scenario: Analyze YAML structure skips comments and finds templates
Given I have YAML with mixed content for structure analysis:
"""
# comment
root:
value: static
{{ inline_var }}
{% for x in items %}
- name: {{ x }}
{% endfor %}
"""
When I analyze the YAML structure
Then template blocks and inline templates should be located
Scenario: Load file wrapper processes templates from disk
Given I have a temporary YAML file with template content:
"""
agent:
name: "{{ name }}"
role: static
"""
When I load the YAML file with context data
Then the file load result should merge templated values
Scenario: Rendering list template raises mapping ValueError
Given I have a templated YAML that renders to a list root:
"""
{% for item in items %}
- {{ item }}
{% endfor %}
"""
And I have a template context with list items
When I render the YAML expecting a non-mapping error
Then a ValueError should indicate rendered YAML must be a mapping
Scenario: Malformed rendered YAML triggers fallback fix path
Given I have templated YAML that renders invalid mappings:
"""
config: {{ messy_value }}
"""
And I have a messy value context for rendering
When I render the malformed YAML content
Then the parser should attempt fixes then raise YAML error
Scenario: Deferred templated list raises mapping ValueError
Given I have a deferred YAML template that becomes a list:
"""
- "{{ value }}"
"""
When I load YAML content without context for deferred rendering
Then a ValueError should be raised for non-mapping deferred templates
Scenario: Parse retry succeeds after YAML error
Given I have YAML with a placeholder causing parse retry:
"""
result: {{ value }}
"""
And I have a template context for retry parsing
When I render YAML with a transient YAML error on first parse
Then the rendering retry should yield a mapping result
Scenario: Parse retry raises ValueError when repaired YAML is not a mapping
Given I have YAML with a placeholder causing parse retry:
"""
result: {{ value }}
"""
And I have a template context for retry parsing
When rendering retry returns non-mapping after YAML error
Then a ValueError should be raised for non-mapping retry
Scenario: Fix common YAML issues splits multiple colons
Given I have inline YAML with multiple colons on one line:
"""
config: key1: val1 key2: val2
"""
When I apply the common YAML fixes
Then the multi-colon line should be split into nested entries
Scenario: Synthetic postprocess respects single-colon guard
Given I have synthetic YAML content for postprocess:
"""
config: alpha beta: gamma
"""
When I postprocess using a single-colon synthetic line
Then the synthetic postprocess should keep the line intact
+14 -8
View File
@@ -651,6 +651,13 @@ All 10 ADRs have been created in `docs/architecture/decisions/`:
- 2025-12-19: Actor persistence primitives verified in codebase: actors table + default index present via `alembic/versions/c3d9b3d0cf3e_add_actors_table.py:1` and SQLAlchemy model carries default/built-in flags at `src/cleveragents/infrastructure/database/models.py:152`; repository enforces built-in/default guards and exposes default setters in `src/cleveragents/infrastructure/database/repositories.py:582` and `src/cleveragents/infrastructure/database/repositories.py:661`; UnitOfWork and DI wire the repository into services at `src/cleveragents/infrastructure/database/unit_of_work.py:170` and `src/cleveragents/application/container.py:85`; ActorService normalization plus default handling lives at `src/cleveragents/application/services/actor_service.py:22`. No gaps found for Stage 7.5 persistence prerequisite; next work is adapter import + CLI flag changes.
**2025-12-25: V2 actor config parsing (unchanged format) + provenance recorded**
- Ported the full v2 YAML config loader into `src/cleveragents/actor/config.py:37-211`, including Jinja2 template handling (via `YAMLTemplateEngine`), placeholder restoration, env-var interpolation with defaults/typing, and explicit error wrapping for malformed YAML; the actor config format is identical to v2—no new formats or routing styles were added.
- Captured v2 reference commit `10ad86df6069423955e42beacbdf7e36dfa779da` for actor parser provenance; future deviations must reconcile against this tag to keep the v2 format intact.
- Added dependency `jinja2>=3.1.0` (project deps) to match the v2 parser requirements.
- Behave coverage added in `features/actor_config_coverage.feature:64-96` with steps in `features/steps/actor_config_steps.py:138-151` validating provider/model inference, unsafe propagation, graph_descriptor keys, option merge behavior, and YAML parse error surfaces (all in v2 format).
- Robot integration added via `robot/helper_actor_config.py:1-29` and `robot/actor_configuration.robot:1-41` to assert provider/model/graph key extraction from v2-format YAML configs end-to-end.
**2025-12-09: Provider streaming normalization + Behave coverage refresh**
- `PlanService.generate_plan_streaming` now consumes the new provider iterator contract, normalizes nested `__end__/response` payloads, validates streamed `Change` objects, persists token counts, and still emits the legacy CLI end event for compatibility (`src/cleveragents/application/services/plan_service.py:858`).
- `LangChainChatProvider.stream_changes` (plus both mock providers) now yield LangGraph workflow events followed by `{"__end__": {"response": ProviderResponse}}`, guaranteeing a consistent exit shape and structured usage logging (`src/cleveragents/providers/llm/langchain_chat_provider.py:154`, `features/mocks/langchain_mock_provider.py:269`, `features/mocks/mock_ai_provider.py:218`).
@@ -4443,12 +4450,12 @@ If you can do all of the above by end of Day 1, you're on track!
- [X] Add actor persistence primitives (database schema/migration + repository + DI wiring).
- [X] Create `src/cleveragents/actor/` package skeleton (module and `__init__`) to host the ported v2 logic and register with the DI container.
- [X] Check out git tag `v2` as a read-only worktree at `./v2` (reference-only, same rule as `./plandex/`; no imports into v3 code).
- [ ] Port the relevant Python from the git `v2` tag into `src/cleveragents/actor/` as first-class v3 code (no `v2` references in code or comments); pin the referenced commit hash in **Phase 2 Notes**; add any new dependencies to `[project.optional-dependencies.actors]`; keep package `__init__` files side-effect free so the actor package controls initialization.
- [ ] Capture the git `v2` commit hash for actor parser provenance and reconcile deviations once the tag is available (add to Phase 2 Notes).
- [ ] Fix record git `v2` actor commit hash once repository tag access is available (currently unavailable in this workspace).
- [X] Port the relevant Python from the git `v2` tag into `src/cleveragents/actor/` as first-class v3 code (no `v2` references in code or comments); pin the referenced commit hash in **Phase 2 Notes**; add any new dependencies to `[project.optional-dependencies.actors]`; keep package `__init__` files side-effect free so the actor package controls initialization.
- [X] Capture the git `v2` commit hash for actor parser provenance and reconcile deviations once the tag is available (add to Phase 2 Notes).
- [X] Fix record git `v2` actor commit hash once repository tag access is available (currently unavailable in this workspace).
- [X] Port the actor-configuration parser/runner from the git `v2` tag into the actor package so it emits `graph_descriptor`, provider/model requirements, normalized options, and `unsafe`; map its flags to a typed Pydantic config model; bind it to the existing provider registry and `ContextService` so all context/state flows through current lifecycles, inject package-produced initial context variables before graph execution, and ensure all call sites use actor names instead of provider/model pairs.
- [X] Recover or define the canonical actor configuration schema/graph descriptor (from the git `v2` tag or a reconstructed spec) to drive the parser, hashing rules, and validation for Stage 7.5.
- [X] Confirm actor config parser is functionally identical to v2 format (no new formats/routing), with the full v2 loader behavior ported into v3.
- [X] Implement an actor registry backed by the config DB with schema: `name`, `config_blob` (canonical JSON/YAML, no file path), `config_hash` (content hash), `graph_descriptor`, `unsafe` (bool), `created_at`, `updated_at`, `default_actor`; generate built-ins from the provider registry (`<provider>/<model>`) at startup as read-only rows; enforce `local/<id>` naming for customs; block removal when target is default; expose CRUD via repository + service layer; add migration to create/modify registry tables and the default pointer.
- [X] Wire CLI: `actor add --name <id> --config <path> [--unsafe]` (auto-prefix `local/`, compute hash, persist blob, require `--unsafe` when the actor config is marked unsafe), `actor update --name <id> [--config <path>] [--unsafe|--safe] [--set-default]` (mutually exclusive unsafe/safe; allow default change without new config), `actor remove local/<id>` (error if default; ensure record exists), `actor list` (built-in/custom, default marker, unsafe flag, hash, created/updated timestamps), `actor show <name>` (unsafe flag, stored hash, graph summary, provider/model requirements, config excerpt). Reject names missing required prefixes and enforce unsafe/safe exclusivity.
- [X] Update `chat`/`plan` and any other provider/model entry points to require `--actor` (remove `--model/--provider` flags and help text), honor `default_actor` when flag is omitted, retain git v2 tag's `run` flags (`--context`, `--load-context`, etc.) via `ContextService`, emit warnings at verbosity ≥ warning for unsafe actors (runtime does not require `--unsafe`), and allow built-in or custom actors as default. Ensure selection resolves to provider registry entries with per-model options forwarded and package-produced graph descriptors injected into execution, and reject raw provider/model usage in favor of actor names.
@@ -4457,16 +4464,15 @@ If you can do all of the above by end of Day 1, you're on track!
- [ ] Port git `v2` tag API/CLI documentation relevant to actors/configuration into docs; scrub `--model/--provider` references and any `v2` naming; update CLI help/man pages and `agents --help` output to describe `--actor`, unsafe semantics, default resolution, and removal guards.
- [ ] Delete the temporary `./v2` reference directory once Stage 7.5 actor porting and documentation are complete (final cleanup step).
- [ ] Document:
- [ ] Update **Phase 2 Notes**, README, and docs (actor/CLI pages) with actor naming rules (`<provider>/<model>` built-ins, `local/<id>` customs), registry storage in config DB (blob + hash + unsafe + graph_descriptor + timestamps + default pointer), default selection behavior, unsafe semantics (`--unsafe`/`--safe` exclusivity), removal guard for defaults, built-in immutability, and context flag parity with v2.
- [ ] Document actor command examples for add/update/set-default/remove/list/show and chat/plan usage, including sample warnings for unsafe actors at verbosity ≥ warning, default resolution examples, failure cases (missing config, missing prefix, attempting to remove default, unsafe flag required), and note that runtime does not require `--unsafe` to execute.
- [ ] Update **Phase 2 Notes**, README, and docs (actor/CLI pages) with actor naming rules (`<provider>/<model>` built-ins, `local/<id>` customs), registry storage in config DB (blob + hash + unsafe + graph_descriptor + timestamps + default pointer), default selection behavior, unsafe semantics (`--unsafe`/`--safe` exclusivity), removal guard for defaults, built-in immutability, context flag parity with v2, and the requirement that actor config files remain in the exact v2 format (no new formats/routing).
- [ ] Document actor command examples for add/update/set-default/remove/list/show and chat/plan usage, including sample warnings for unsafe actors at verbosity ≥ warning, default resolution examples, failure cases (missing config, missing prefix, attempting to remove default, unsafe flag required), runtime not requiring `--unsafe`, and explicit statements that actor config files must conform to the v2 format without extensions or alternative schemas.
- [ ] Update architecture references/ADRs (ADR-008/ADR-011 or addenda) to capture actor registry boundaries, actor package responsibilities, dependency on provider registry + `ContextService`, default pointer storage, built-in immutability, and removal of `--model/--provider` in favor of `--actor`.
- [ ] Include migrated API documentation from git's v2 tag describing actor configuration fields, supported options, graph descriptor semantics, unsafe detection rules, and how actor package outputs map to provider/model selection and LangGraph invocation.
- [ ] Tests:
- [X] Add v2-format actor config parsing coverage (Behave + Robot) for YAML inference of provider/model/graph/options (format unchanged between v2 and v3).
- [ ] Port git v2's tag unit coverage into Behave features covering actor configuration parsing (valid/invalid blobs, option normalization), actor package outputs (graph_descriptor, provider/model requirements), unsafe detection, registry CRUD (hash stability, default guard, removal requiring `local/<id>`), built-in enumeration, default selection, warning emission at verbosity ≥ warning, and chat/plan flows with `--actor` plus context flags.
- [ ] Migrate existing provider/model Behave and Robot suites (plan CLI, plan_service/provider_registry overrides, streaming/auto-debug flows) to actor-only flags and expectations so provider/model paths are removed without coverage regressions.
- [ ] Port git v2 tag's integration/e2e coverage into Robot suites for actor add/update/remove/list/show, default-actor deletion guard, unsafe warning display, chat/plan end-to-end with custom and built-in actors, and persistence of context across invocations with `--context`/`--load-context`; include DB persistence checks for blob/hash/unsafe/default fields.
- [ ] Add Behave/Robot tests ensuring config DB stores canonical blobs (no path reliance), hash only changes on update, `--unsafe`/`--safe` mutual exclusivity enforcement, default fallback when `--actor` is omitted, built-in immutability, provider-model option forwarding, and warning emission without requiring runtime `--unsafe`.
- [ ] Update nox sessions to include new actor Behave/Robot suites; replace/remove `--model/--provider` coverage with actor-based tests; ensure coverage stays ≥85% and pyright remains clean; verify docs build for ported API references and CLI help updates.
- [ ] Stage 8: Async Infrastructure
+1
View File
@@ -38,6 +38,7 @@ dependencies = [
"structlog>=24.0.0",
# Configuration
"pyyaml>=6.0.2",
"jinja2>=3.1.0",
"python-dotenv>=1.0.0",
# Phase 2: Database & ORM
"sqlalchemy>=2.0.0",
+39
View File
@@ -0,0 +1,39 @@
*** Settings ***
Library OperatingSystem
Library Collections
Library Process
*** Test Cases ***
V2 Actor Config Produces Provider And Graph Descriptor
${config}= Set Variable ${OUTPUT DIR}/v2_actor.yaml
${content}= Catenate SEPARATOR=\n
... cleveragents:
... ${SPACE*2}default_router: main_router
... agents:
... ${SPACE*2}paper_writer:
... ${SPACE*4}type: llm
... ${SPACE*4}config:
... ${SPACE*6}provider: openai
... ${SPACE*6}model: gpt-4
... ${SPACE*6}unsafe: true
... ${SPACE*6}options:
... ${SPACE*8}temperature: 0.5
... routes:
... ${SPACE*2}main_router:
... ${SPACE*4}type: stream
... ${SPACE*4}operators:
... ${SPACE*6}- type: map
... ${SPACE*8}params:
... ${SPACE*10}agent: paper_writer
... ${SPACE*4}publications:
... ${SPACE*6}- __output__
Create File ${config} ${content}
${result}= Run Process ${CURDIR}/../.nox/integration_tests-3-13/bin/python robot/helper_actor_config.py ${config} stdout=PIPE stderr=PIPE
Should Be Equal As Integers ${result.rc} 0
${payload}= Evaluate __import__('json').loads('''${result.stdout.strip()}''')
Should Be Equal ${payload['provider']} openai
Should Be Equal ${payload['model']} gpt-4
Should Be True ${payload['unsafe']}
List Should Contain Value ${payload['graph_keys']} agents
List Should Contain Value ${payload['graph_keys']} routes
Should Be Equal As Numbers ${payload['options']['temperature']} 0.5
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
import json
import sys
from pathlib import Path
from cleveragents.actor.config import ActorConfiguration
def main() -> None:
if len(sys.argv) < 2:
raise SystemExit("config path required")
config_path = Path(sys.argv[1])
config = ActorConfiguration.from_file(path=config_path)
payload = {
"provider": config.provider,
"model": config.model,
"unsafe": config.unsafe,
"graph_keys": sorted(config.graph_descriptor.keys())
if config.graph_descriptor
else [],
"options": config.options,
}
print(json.dumps(payload))
if __name__ == "__main__":
main()
+205 -18
View File
@@ -1,13 +1,15 @@
"""Actor configuration models and parsing utilities."""
from __future__ import annotations
import json
import os
import re
from pathlib import Path
from typing import Any, cast
from pydantic import BaseModel, Field
from cleveragents.actor.yaml_template_engine import YAMLTemplateEngine
try: # Optional dependency; present in dev/test extras
import yaml
except Exception: # pragma: no cover - defensive optional import
@@ -32,7 +34,7 @@ class ActorConfiguration(BaseModel):
@staticmethod
def load_blob_from_file(config_path: Path) -> dict[str, Any]:
"""Load a configuration blob from a JSON or YAML file."""
"""Load a configuration blob from a JSON or YAML file (v2 parity)."""
path = config_path.expanduser()
if not path.exists():
@@ -41,20 +43,135 @@ class ActorConfiguration(BaseModel):
text = path.read_text()
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
if yaml is None:
raise ValueError("PyYAML is required for YAML actor configs") from exc
try:
data = yaml.safe_load(text)
except Exception as yaml_exc: # pragma: no cover - defensive
raise ValueError(f"Failed to parse config: {yaml_exc}") from yaml_exc
except json.JSONDecodeError:
data = ActorConfiguration._load_v2_yaml_content(text)
else:
if data is None:
return {}
if not isinstance(data, dict):
raise ValueError("Config must be a JSON or YAML object")
return cast(dict[str, Any], data)
if data is None:
return {}
if not isinstance(data, dict):
return data
@staticmethod
def _load_v2_yaml_content(text: str) -> dict[str, Any]:
if yaml is None:
raise ValueError("PyYAML is required for YAML actor configs")
raw: Any
try:
if "{%" in text or "{{" in text:
protected = re.sub(r"{{", "<<<TEMPLATE_START>>>", text)
protected = re.sub(r"}}", "<<<TEMPLATE_END>>>", protected)
protected = re.sub(r"{%", "<<<BLOCK_START>>>", protected)
protected = re.sub(r"%}", "<<<BLOCK_END>>>", protected)
engine = YAMLTemplateEngine()
minimal_context: dict[str, dict[str, Any]] = {
"context": {
"paper_details": {
"topic": "",
"length": "",
"audience": "",
"publication": "",
"format": "",
"other": "",
},
"brainstorming_summary": "",
"vetting_sources": list[str](),
"table_of_contents": "",
"deep_research_sources": "",
"current_section_to_write": "",
"paper_content": dict[str, Any](),
"final_paper_text": "",
"proofread_paper": "",
}
}
raw = engine.load_string(protected, context=minimal_context)
raw = ActorConfiguration._restore_template_syntax(raw)
else:
raw = yaml.safe_load(text)
except Exception as exc: # pragma: no cover - defensive
raise ValueError(f"Failed to parse config: {exc}") from exc
if raw is None:
raw = {}
if not isinstance(raw, dict):
raise ValueError("Config must be a JSON or YAML object")
return cast(dict[str, Any], data)
interpolated = ActorConfiguration._interpolate_env_vars(raw)
if not isinstance(interpolated, dict):
raise ValueError("Config must be a JSON or YAML object")
return cast(dict[str, Any], interpolated)
@staticmethod
def _restore_template_syntax(config: Any) -> Any:
if isinstance(config, dict):
config_dict = cast(dict[str, Any], config)
restored: dict[str, Any] = {}
for key, value in config_dict.items():
if key == "system_prompt" and isinstance(value, str):
updated = value.replace("<<<TEMPLATE_START>>>", "{{").replace(
"<<<TEMPLATE_END>>>", "}}"
)
updated = updated.replace("<<<BLOCK_START>>>", "{%")
updated = updated.replace("<<<BLOCK_END>>>", "%}")
restored[key] = updated
else:
restored[key] = ActorConfiguration._restore_template_syntax(value)
return restored
if isinstance(config, list):
config_list = cast(list[Any], config)
return [
ActorConfiguration._restore_template_syntax(item)
for item in config_list
]
return config
@staticmethod
def _interpolate_env_vars(config: Any) -> Any:
if isinstance(config, dict):
config_dict = cast(dict[str, Any], config)
return {
k: ActorConfiguration._interpolate_env_vars(v)
for k, v in config_dict.items()
}
if isinstance(config, list):
config_list = cast(list[Any], config)
return [ActorConfiguration._interpolate_env_vars(i) for i in config_list]
if isinstance(config, str):
env_var_pattern = r"\${([A-Za-z0-9_]+)(?::([^}]*))?\}"
def replace_env_var(match: re.Match[str]) -> str:
env_var = match.group(1)
default_value = match.group(2) if match.group(2) is not None else None
env_value = os.environ.get(env_var)
if env_value is None:
if default_value is not None:
if default_value.lower() in {"true", "false"}:
return str(default_value.lower() == "true")
if default_value.lstrip("-").isdigit():
return default_value
return default_value
raise ValueError(f"Environment variable '{env_var}' is not set")
return env_value
substituted = re.sub(env_var_pattern, replace_env_var, config)
lowered = substituted.lower()
if lowered in {"true", "false"}:
return lowered == "true"
if substituted.lstrip("-").isdigit():
return int(substituted)
if (
substituted.lstrip("-").replace(".", "", 1).isdigit()
and substituted.count(".") == 1
):
return float(substituted)
return substituted
return config
@classmethod
def from_file(
@@ -96,12 +213,18 @@ class ActorConfiguration(BaseModel):
data: dict[str, Any] = dict(blob) if isinstance(blob, dict) else {}
v2_provider, v2_model, v2_graph, v2_unsafe = cls._extract_v2_actor(data)
v2_options = cls._extract_v2_options(data)
resolved_provider = (
provider or data.get("provider") or data.get("provider_type")
provider or data.get("provider") or data.get("provider_type") or v2_provider
)
resolved_model = model or data.get("model") or data.get("model_id")
resolved_model = model or data.get("model") or data.get("model_id") or v2_model
resolved_graph = (
graph_descriptor or data.get("graph_descriptor") or data.get("graph")
graph_descriptor
or data.get("graph_descriptor")
or data.get("graph")
or v2_graph
)
options_raw = data.get("options")
@@ -111,12 +234,14 @@ class ActorConfiguration(BaseModel):
merged_options: dict[str, Any] = {}
if default_options:
merged_options.update(default_options)
if v2_options:
merged_options.update(v2_options)
if options_value:
merged_options.update(options_value)
if option_overrides:
merged_options.update(option_overrides)
resolved_unsafe = bool(data.get("unsafe", False)) or unsafe
resolved_unsafe = bool(data.get("unsafe", False)) or v2_unsafe or unsafe
if not resolved_provider:
raise ValueError("provider is required")
@@ -138,3 +263,65 @@ class ActorConfiguration(BaseModel):
options=merged_options,
unsafe=resolved_unsafe,
)
@staticmethod
def _extract_v2_actor(
data: dict[str, Any],
) -> tuple[str | None, str | None, dict[str, Any] | None, bool]:
"""Derive provider/model/graph from the v2 YAML actor config format."""
agents_obj = data.get("agents")
if isinstance(agents_obj, dict) and agents_obj:
agents_dict = cast(dict[str, Any], agents_obj)
for first_agent, first_entry in agents_dict.items():
if isinstance(first_entry, dict):
entry_dict = cast(dict[str, Any], first_entry)
config_block = entry_dict.get("config")
if isinstance(config_block, dict):
config_dict = cast(dict[str, Any], config_block)
provider_value = config_dict.get("provider") or config_dict.get(
"provider_type"
)
model_value = config_dict.get("model") or config_dict.get(
"model_id"
)
unsafe_flag = bool(config_dict.get("unsafe", False))
descriptor: dict[str, Any] = {
"agent": first_agent,
"agents": agents_dict,
}
for key in (
"routes",
"merges",
"templates",
"cleveragents",
):
if key in data and data[key] is not None:
descriptor[key] = data[key]
return (
str(provider_value) if provider_value else None,
str(model_value) if model_value else None,
descriptor,
unsafe_flag,
)
break
return None, None, None, False
@staticmethod
def _extract_v2_options(data: dict[str, Any]) -> dict[str, Any] | None:
"""Extract option defaults from the v2 actor config format."""
agents_obj = data.get("agents")
if isinstance(agents_obj, dict) and agents_obj:
agents_dict = cast(dict[str, Any], agents_obj)
for _, first_entry in agents_dict.items():
if isinstance(first_entry, dict):
entry_dict = cast(dict[str, Any], first_entry)
config_block = entry_dict.get("config")
if isinstance(config_block, dict):
config_dict = cast(dict[str, Any], config_block)
options = config_dict.get("options")
if isinstance(options, dict):
return cast(dict[str, Any], options)
break
return None
@@ -0,0 +1,242 @@
"""YAML Template Engine with inline Jinja2 support (ported from v2)."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Callable, cast
import yaml
from jinja2 import Environment
logger = logging.getLogger(__name__)
class YAMLTemplateEngine:
"""Complete solution for inline Jinja2 templates in YAML (v2 parity)."""
def __init__(self) -> None:
# Configure Jinja2 for YAML-friendly output
self.env = Environment(
block_start_string="{%",
block_end_string="%}",
variable_start_string="{{",
variable_end_string="}}",
comment_start_string="{#",
comment_end_string="#}",
trim_blocks=False,
lstrip_blocks=False,
keep_trailing_newline=True,
)
# Add custom filters for YAML
self.env.filters["yaml"] = self._yaml_filter
self.env.filters["indent"] = self._indent_filter
self.env.filters["sum"] = self._sum_filter
self.env.filters["selectattr"] = cast(
Callable[..., Any], self._selectattr_filter
)
def load_file(
self, file_path: Path, context: dict[str, Any] | None = None
) -> dict[str, Any]:
with open(file_path, encoding="utf-8") as f:
content = f.read()
return self.load_string(content, context)
def load_string(
self, yaml_content: str, context: dict[str, Any] | None = None
) -> dict[str, Any]:
if "{%" not in yaml_content and "{{" not in yaml_content:
loaded = yaml.safe_load(yaml_content)
if not isinstance(loaded, dict):
raise ValueError(
f"Expected YAML to load as dict, got {type(loaded).__name__}"
)
return cast(dict[str, Any], loaded)
if context is not None:
return self._render_and_parse(yaml_content, context)
return self._prepare_for_deferred_rendering(yaml_content)
def _render_and_parse(
self, yaml_content: str, context: dict[str, Any]
) -> dict[str, Any]:
processed_content = self._preprocess_for_rendering(yaml_content)
full_context = self._create_render_context(context)
template = self.env.from_string(processed_content)
rendered = template.render(**full_context)
fixed_yaml = self._postprocess_rendered_yaml(rendered)
try:
parsed = yaml.safe_load(fixed_yaml)
if not isinstance(parsed, dict):
raise ValueError(
f"Expected YAML to parse as dict, got {type(parsed).__name__}"
)
return cast(dict[str, Any], parsed)
except yaml.YAMLError as exc:
logger.error("Failed to parse rendered YAML: %s", exc)
logger.debug("Rendered YAML:\n%s", fixed_yaml)
fixed_yaml = self._fix_common_yaml_issues(fixed_yaml)
parsed = yaml.safe_load(fixed_yaml)
if not isinstance(parsed, dict):
raise ValueError(
f"Expected YAML to parse as dict, got {type(parsed).__name__}"
) from exc
return cast(dict[str, Any], parsed)
def _preprocess_for_rendering(self, content: str) -> str:
lines = content.split("\n")
processed_lines: list[str] = []
for line in lines:
if "{%" in line and "for" in line:
indent = len(line) - len(line.lstrip())
processed_lines.append(line)
processed_lines.append(f"{' ' * indent}{{# indent: {indent} #}}")
else:
processed_lines.append(line)
return "\n".join(processed_lines)
def _postprocess_rendered_yaml(self, content: str) -> str:
lines = content.split("\n")
fixed_lines: list[str] = []
for i, line in enumerate(lines):
if 0 < i < len(lines) - 1 and line.strip() == "":
next_line = lines[i + 1] if i + 1 < len(lines) else ""
if next_line and not next_line.startswith(" "):
continue
if ":" in line and line.count(":") == 1:
parts = line.split(":", 1)
if len(parts) == 2:
key_part = parts[0]
value_part = parts[1].strip()
if value_part and " " in value_part and ":" not in value_part:
value_words = value_part.split()
if len(value_words) > 1 and any(
":" in w for w in value_words[1:]
):
fixed_lines.append(f"{key_part}: {value_words[0]}")
remaining = " ".join(value_words[1:])
indent = len(key_part) - len(key_part.lstrip())
fixed_lines.append(f"{' ' * indent}{remaining}")
continue
fixed_lines.append(line)
return "\n".join(fixed_lines)
def _fix_common_yaml_issues(self, content: str) -> str:
lines = content.split("\n")
fixed_lines: list[str] = []
for line in lines:
if line.count(":") > 1:
parts = line.split(":", 1)
if len(parts) == 2:
indent = len(parts[0]) - len(parts[0].lstrip())
fixed_lines.append(f"{parts[0]}:")
remaining = parts[1].strip()
if remaining:
tokens = remaining.split()
current_line = ""
for token in tokens:
if ":" in token and token.endswith(":"):
if current_line:
fixed_lines.append(
f"{' ' * (indent + 2)}{current_line}"
)
fixed_lines.append(f"{' ' * (indent + 2)}{token}")
current_line = ""
else:
current_line += f" {token}" if current_line else token
if current_line:
fixed_lines.append(f"{' ' * (indent + 2)}{current_line}")
continue
fixed_lines.append(line)
return "\n".join(fixed_lines)
def _prepare_for_deferred_rendering(self, yaml_content: str) -> dict[str, Any]:
return self._simple_template_extraction(yaml_content)
def _analyze_yaml_structure(self, content: str) -> dict[str, Any]:
lines = content.split("\n")
structure: dict[str, Any] = {
"template_blocks": [],
"inline_templates": [],
"hierarchy": [],
}
current_path: list[tuple[str, int]] = []
for i, line in enumerate(lines):
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
indent = len(line) - len(line.lstrip())
while current_path and indent <= current_path[-1][1]:
current_path.pop()
if ":" in line:
key = line.split(":", 1)[0].strip()
current_path.append((key, indent))
structure["hierarchy"].append(
{
"line": i,
"key": key,
"indent": indent,
"path": [p[0] for p in current_path],
}
)
if "{%" in line:
structure["template_blocks"].append(i)
if "{{" in line:
structure["inline_templates"].append(i)
return structure
def _simple_template_extraction(self, content: str) -> dict[str, Any]:
rendered = content
try:
parsed = yaml.safe_load(rendered)
if not isinstance(parsed, dict):
raise ValueError(
f"Expected YAML to load as dict, got {type(parsed).__name__}"
)
return cast(dict[str, Any], parsed)
except yaml.YAMLError as exc:
logger.error("Failed to parse YAML with templates: %s", exc)
logger.debug("YAML content with templates:\n%s", rendered)
raise
@staticmethod
def _yaml_filter(value: Any) -> str:
return yaml.dump(value, default_flow_style=False).strip()
@staticmethod
def _indent_filter(value: Any, spaces: int = 2) -> str:
return "\n".join(" " * spaces + line for line in str(value).split("\n"))
@staticmethod
def _sum_filter(value: Any) -> Any:
try:
return sum(value)
except Exception as exc: # pragma: no cover - defensive
raise ValueError(f"Cannot sum value: {value}") from exc
@staticmethod
def _selectattr_filter(
sequence: list[Any], attr: str, default: Any = None
) -> list[Any]:
results: list[Any] = []
for item in sequence:
if hasattr(item, attr):
results.append(getattr(item, attr))
else:
results.append(default)
return results
@staticmethod
def _create_render_context(context: dict[str, Any]) -> dict[str, Any]:
nested_context = context.get("context")
nested_mapping = (
cast(dict[str, Any], nested_context)
if isinstance(nested_context, dict)
else {}
)
base_context: dict[str, Any] = {"context": nested_mapping.copy()}
return base_context | context