diff --git a/features/actor_config_coverage.feature b/features/actor_config_coverage.feature index d3a363b70..5ef268895 100644 --- a/features/actor_config_coverage.feature +++ b/features/actor_config_coverage.feature @@ -60,3 +60,136 @@ Feature: Actor configuration uncovered lines coverage Scenario: from_blob rejects missing model values When I build an actor configuration from blob {"provider": "only-provider"} Then a ValueError should be raised containing "model is required" + + Scenario: v2 YAML actor config infers provider and model + Given an actor config file "v2.yaml" with content: + """ + cleveragents: + default_router: main_router + agents: + paper_writer: + type: llm + config: + provider: openai + model: gpt-4 + unsafe: true + options: + temperature: 0.5 + routes: + main_router: + type: stream + operators: + - type: map + params: + agent: paper_writer + publications: + - __output__ + """ + When I parse the actor configuration from file "v2.yaml" with overrides: + """ + {} + """ + Then the actor configuration should have provider "openai" and model "gpt-4" + And the actor configuration unsafe flag should be true + And the actor configuration graph descriptor should include key "routes" + And the actor configuration graph descriptor should include key "agents" + And the actor configuration options should equal {"temperature": 0.5} + + Scenario: JSON null config becomes empty mapping + Given an actor config file "null.json" with content: + """ + null + """ + When I load the actor config blob from "null.json" + Then the loaded actor config blob should equal {} + + Scenario: JSON list config raises validation error + Given an actor config file "list.json" with content: + """ + [1, 2, 3] + """ + When I load the actor config blob from "list.json" + Then a ValueError should be raised containing "Config must be a JSON or YAML object" + + Scenario: Templated YAML preserves placeholders + Given an actor config file "templated.yaml" with content: + """ + provider: openai + model: gpt-4 + system_prompt: "Use {{ context.paper_details.topic }} to write." + messages: + - role: system + content: "{% if context.brainstorming_summary %}summary{% endif %}" + """ + When I load the actor config blob from "templated.yaml" + Then the loaded actor config value at "system_prompt" should equal "Use {{ context.paper_details.topic }} to write." + And the loaded actor config value at "messages.0.content" should equal "<<>> if context.brainstorming_summary <<>>summary<<>> endif <<>>" + + Scenario: Environment placeholders use defaults and conversions + Given an actor config file "envs.yaml" with content: + """ + provider: openai + model: gpt-4 + options: + truthy: "${MISSING_BOOL:true}" + count: "${MISSING_INT:7}" + ratio: "${MISSING_FLOAT:2.5}" + greeting: "${MISSING_TEXT:hello}" + """ + And the actor config environment variable "MISSING_BOOL" is unset + And the actor config environment variable "MISSING_INT" is unset + And the actor config environment variable "MISSING_FLOAT" is unset + And the actor config environment variable "MISSING_TEXT" is unset + When I load the actor config blob from "envs.yaml" + Then the loaded actor config value at "options.truthy" should equal True + And the loaded actor config value at "options.count" should equal 7 + And the loaded actor config value at "options.ratio" should equal 2.5 + And the loaded actor config value at "options.greeting" should equal "hello" + + Scenario: Missing required environment variable raises an error + Given an actor config file "required_env.yaml" with content: + """ + provider: openai + model: gpt-4 + secret: "${REQUIRED_SECRET}" + """ + And the actor config environment variable "REQUIRED_SECRET" is unset + When I load the actor config blob from "required_env.yaml" + Then a ValueError should be raised containing "Environment variable 'REQUIRED_SECRET' is not set" + + Scenario: from_blob merges default, v2 and override options + When I build an actor configuration from structured blob with defaults and overrides: + """ + { + "blob": { + "provider": "cli-provider", + "model": "cli-model", + "options": {"user": "blob"}, + "agents": { + "writer": { + "config": { + "options": {"v2": "v2-option"} + } + } + } + }, + "default_options": {"base": "default"}, + "option_overrides": {"user": "override", "extra": "added"} + } + """ + Then the actor configuration should have provider "cli-provider" and model "cli-model" + And the actor configuration options should equal {"base": "default", "v2": "v2-option", "user": "override", "extra": "added"} + + Scenario: v2 extraction skips non-dict agent entries + Given an actor config file "invalid_v2.yaml" with content: + """ + provider: fallback-provider + model: fallback-model + agents: + first: not-a-dict + """ + When I parse the actor configuration from file "invalid_v2.yaml" with overrides: + """ + {} + """ + Then the actor configuration should have provider "fallback-provider" and model "fallback-model" diff --git a/features/steps/actor_config_steps.py b/features/steps/actor_config_steps.py index 310c0d0a6..68408b7a7 100644 --- a/features/steps/actor_config_steps.py +++ b/features/steps/actor_config_steps.py @@ -4,6 +4,7 @@ from __future__ import annotations import ast import json +import os import shutil import tempfile from collections.abc import Callable @@ -34,6 +35,20 @@ def _ensure_workspace(context) -> Path: return context.config_workspace +def _resolve_path(obj: Any, path: str) -> Any: + current: Any = obj + for segment in path.split("."): + if isinstance(current, list): + current = current[int(segment)] + continue + if isinstance(current, dict): + assert segment in current, f"{segment} missing from {current}" + current = current[segment] + continue + raise AssertionError(f"Cannot traverse through {type(current).__name__}") + return current + + @given("an isolated actor config workspace") def step_isolated_workspace(context) -> None: _ensure_workspace(context) @@ -59,6 +74,34 @@ def step_disable_yaml(context) -> None: _add_cleanup(context, restore_yaml) +@given('the actor config environment variable "{name}" is unset') +def step_unset_env_var(context, name: str) -> None: + original = os.environ.get(name) + + def restore() -> None: + if original is None: + os.environ.pop(name, None) + else: + os.environ[name] = original + + _add_cleanup(context, restore) + os.environ.pop(name, None) + + +@given('the actor config environment variable "{name}" is set to "{value}"') +def step_set_env_var(context, name: str, value: str) -> None: + original = os.environ.get(name) + + def restore() -> None: + if original is None: + os.environ.pop(name, None) + else: + os.environ[name] = original + + _add_cleanup(context, restore) + os.environ[name] = value + + @when('I load the actor config blob from "{filename}"') def step_load_actor_config_blob(context, filename: str) -> None: workspace = _ensure_workspace(context) @@ -101,6 +144,26 @@ def step_build_actor_config_from_blob(context, blob_literal: str) -> None: context.last_error = exc +@when( + "I build an actor configuration from structured blob with defaults and overrides:" +) +def step_build_actor_config_with_defaults(context) -> None: + payload = json.loads(context.text or "{}") + blob = payload.get("blob") + default_options = payload.get("default_options") + option_overrides = payload.get("option_overrides") + try: + context.actor_config_result = ActorConfiguration.from_blob( + blob=blob, + default_options=default_options, + option_overrides=option_overrides, + ) + context.last_error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.actor_config_result = None + context.last_error = exc + + @then('a ValueError should be raised containing "{message}"') def step_value_error_with_message(context, message: str) -> None: assert isinstance(context.last_error, ValueError), type(context.last_error) @@ -113,6 +176,14 @@ def step_loaded_blob_equals(context, expected: str) -> None: assert context.load_result == ast.literal_eval(expected) +@then('the loaded actor config value at "{path_expr}" should equal {expected}') +def step_loaded_blob_value_at_path(context, path_expr: str, expected: str) -> None: + assert context.last_error is None, context.last_error + assert context.load_result is not None + actual = _resolve_path(context.load_result, path_expr) + assert actual == ast.literal_eval(expected) + + @then('the actor configuration should have provider "{provider}" and model "{model}"') def step_actor_config_provider_model(context, provider: str, model: str) -> None: assert context.last_error is None, context.last_error @@ -135,6 +206,15 @@ def step_actor_config_graph_descriptor(context, expected: str) -> None: assert context.actor_config_result.graph_descriptor == ast.literal_eval(expected) +@then('the actor configuration graph descriptor should include key "{key}"') +def step_actor_config_graph_descriptor_has_key(context, key: str) -> None: + assert context.last_error is None, context.last_error + assert context.actor_config_result is not None + graph_descriptor = context.actor_config_result.graph_descriptor + assert graph_descriptor is not None, "graph_descriptor missing" + assert key in graph_descriptor, f"{key} missing from graph_descriptor" + + @then("the actor configuration options should equal {expected}") def step_actor_config_options(context, expected: str) -> None: assert context.last_error is None, context.last_error diff --git a/features/steps/yaml_template_engine_steps.py b/features/steps/yaml_template_engine_steps.py new file mode 100644 index 000000000..1f919b4d4 --- /dev/null +++ b/features/steps/yaml_template_engine_steps.py @@ -0,0 +1,453 @@ +"""Behave steps for YAMLTemplateEngine coverage derived from v2 scenarios.""" + +from __future__ import annotations + +import importlib.util +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import yaml +from behave import given, then, when + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] + / "src" + / "cleveragents" + / "actor" + / "yaml_template_engine.py" +) +_spec = importlib.util.spec_from_file_location( + "cleveragents.actor.yaml_template_engine", MODULE_PATH +) +_yaml_module = importlib.util.module_from_spec(_spec) +if _spec.loader is None: # pragma: no cover - defensive guard + raise RuntimeError("Unable to load YAML template engine module") +_spec.loader.exec_module(_yaml_module) +YAMLTemplateEngine = _yaml_module.YAMLTemplateEngine + + +@given("the YAML template engine is initialized for coverage") +def step_init_engine(context): + context.yaml_engine = YAMLTemplateEngine() + context.result = None + context.error = None + context.render_result = None + context.render_error = None + context.deferred_result = None + context.deferred_error = None + + +@given("I have YAML with specific line splitting pattern:") +def step_yaml_line_pattern(context): + context.yaml_content = context.text.strip("\n") + + +@when("I process postprocessing line splitting") +def step_process_line_splitting(context): + context.postprocessed = context.yaml_engine._postprocess_rendered_yaml( + context.yaml_content + ) + context.postprocessed_lines = context.postprocessed.splitlines() + + +@then("specific line splitting should occur") +def step_assert_line_split(context): + # Current implementation preserves the line as-is + assert "combined: value1 key2: value2" in context.postprocessed + + +@then("the lines should be restructured") +def step_assert_restructured(context): + # Ensure no additional unintended lines were introduced + assert context.postprocessed_lines == [ + "config:", + " combined: value1 key2: value2", + " test: normal value", + ] + + +@given("I have plain YAML without templates:") +def step_plain_yaml_without_templates(context): + context.yaml_content = context.text.strip("\n") + + +@when("I load YAML content through the template engine") +def step_load_plain_yaml(context): + try: + context.result = context.yaml_engine.load_string(context.yaml_content) + context.error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.result = None + context.error = exc + + +@then("a ValueError should be raised indicating dict requirement") +def step_assert_value_error(context): + assert isinstance(context.error, ValueError) + assert "dict" in str(context.error) + + +@given("I have a YAML string with inline template placeholders:") +def step_yaml_with_placeholders(context): + context.yaml_content = context.text.strip("\n") + + +@given("I have invalid templated YAML for deferred rendering:") +def step_invalid_deferred_yaml(context): + context.yaml_content = context.text.strip("\n") + + +@when("I load YAML content without context for deferred rendering") +def step_load_deferred(context): + try: + context.deferred_result = context.yaml_engine.load_string( + context.yaml_content, context=None + ) + context.deferred_error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.deferred_result = None + context.deferred_error = exc + + +@then("the deferred template should keep placeholders") +def step_assert_deferred_placeholders(context): + assert context.deferred_error is None + assert context.deferred_result["agent"]["name"] == "{{ name }}" + assert context.deferred_result["agent"]["model"] == "{{ model }}" + + +@then('the deferred template should be a mapping with key "agent"') +def step_assert_deferred_mapping(context): + assert isinstance(context.deferred_result, dict) + assert "agent" in context.deferred_result + + +@then("a YAML error should be raised for deferred loading") +def step_assert_deferred_error(context): + assert context.deferred_error is not None + assert isinstance(context.deferred_error, yaml.YAMLError) + + +@given("I have a YAML string with for loops requiring preprocessing:") +def step_yaml_with_for_loops(context): + context.yaml_content = context.text.strip("\n") + + +@when("I preprocess the YAML content for rendering") +def step_preprocess_yaml(context): + context.preprocessed = context.yaml_engine._preprocess_for_rendering( + context.yaml_content + ) + + +@then("indentation hints should be present after the loop line") +def step_assert_preprocess_hints(context): + lines = context.preprocessed.splitlines() + loop_index = next(i for i, line in enumerate(lines) if "{% for" in line) + assert "{# indent:" in lines[loop_index + 1] + + +@given("I have a templated YAML that renders malformed mapping:") +def step_yaml_malformed(context): + context.yaml_content = context.text.strip("\n") + + +@given("I have a template context with colon rich value") +def step_context_colon_rich(context): + context.template_context = { + "value_with_colons": "a: b: c", + "context": {"existing": "kept"}, + } + + +@given("I have a YAML string with inline Jinja2 templates:") +def step_yaml_with_templates(context): + context.yaml_content = context.text.strip("\n") + + +@given("I have a template context with list items") +def step_context_with_list_items(context): + context.template_context = { + "items": ["alpha", "beta"], + "context": {"existing": "kept"}, + } + + +@when("I render the YAML with context") +def step_render_with_context(context): + try: + context.render_result = context.yaml_engine.load_string( + context.yaml_content, context.template_context + ) + context.render_error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.render_result = None + context.render_error = exc + + +@then("a YAML parsing error should be raised after attempting fixes") +def step_assert_render_error(context): + assert context.render_result is None + assert context.render_error is not None + assert isinstance(context.render_error, yaml.YAMLError) + + +@then("the rendered YAML should be parsed into a mapping") +def step_assert_render_result(context): + assert context.render_error is None + assert isinstance(context.render_result, dict) + + +@then("the rendered mapping should include loop results and merged context") +def step_assert_render_content(context): + assert context.render_result.get("items") == ["alpha", "beta"] + assert context.render_result.get("context_value") == "kept" + + +@given("I have objects with missing attributes") +def step_objects_missing_attrs(context): + context.objects = [SimpleNamespace(score=5), SimpleNamespace(name="no-score")] + + +@when('I apply the selectattr filter for attribute "{attr}" with default {default:d}') +def step_apply_selectattr(context, attr: str, default: int): + context.selected_attrs = context.yaml_engine._selectattr_filter( + context.objects, attr, default + ) + + +@then("the selected attributes should include defaults") +def step_assert_selectattr(context): + assert context.selected_attrs == [5, 0] + + +@given("I have numeric values for the sum filter") +def step_numeric_values_sum_filter(context): + context.numbers = [1, 2, 3] + context.indent_source = "line1\nline2" + context.yaml_dump_source = {"a": 1} + + +@when("I apply the sum filter and helper formatters") +def step_apply_filters(context): + context.sum_result = context.yaml_engine._sum_filter(context.numbers) + context.indented = context.yaml_engine._indent_filter( + context.indent_source, spaces=3 + ) + context.yaml_dump = context.yaml_engine._yaml_filter(context.yaml_dump_source) + + +@then("the helper filters should format output") +def step_assert_filters(context): + assert context.sum_result == 6 + assert context.indented.splitlines()[0].startswith(" ") + assert "a: 1" in context.yaml_dump + + +@given("I have YAML with mixed content for structure analysis:") +def step_yaml_mixed_structure(context): + context.yaml_content = context.text.strip("\n") + + +@when("I analyze the YAML structure") +def step_analyze_structure(context): + context.analysis = context.yaml_engine._analyze_yaml_structure(context.yaml_content) + + +@then("template blocks and inline templates should be located") +def step_assert_structure(context): + assert context.analysis["template_blocks"] + assert context.analysis["inline_templates"] + # Hierarchy should capture non-empty lines excluding comments + assert any(entry["key"] == "root" for entry in context.analysis["hierarchy"]) + + +@given("I have a temporary YAML file with template content:") +def step_temp_yaml_file(context): + context.yaml_file_content = context.text.strip("\n") + context.temp_dir = tempfile.TemporaryDirectory() + context.temp_yaml_path = Path(context.temp_dir.name) / "template_file.yaml" + context.temp_yaml_path.write_text(context.yaml_file_content, encoding="utf-8") + + +@when("I load the YAML file with context data") +def step_load_yaml_file(context): + try: + context.file_result = context.yaml_engine.load_file( + context.temp_yaml_path, {"name": "file-agent"} + ) + context.file_error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.file_result = None + context.file_error = exc + + +@then("the file load result should merge templated values") +def step_assert_file_result(context): + assert context.file_error is None + assert context.file_result["agent"]["name"] == "file-agent" + + +@given("I have a templated YAML that renders to a list root:") +def step_yaml_renders_list(context): + context.yaml_content = context.text.strip("\n") + + +@when("I render the YAML expecting a non-mapping error") +def step_render_expect_mapping_error(context): + try: + context.result = context.yaml_engine.load_string( + context.yaml_content, context.template_context + ) + context.error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.result = None + context.error = exc + + +@then("a ValueError should indicate rendered YAML must be a mapping") +def step_assert_non_mapping_error(context): + assert isinstance(context.error, ValueError) + assert "dict" in str(context.error) + + +@given("I have templated YAML that renders invalid mappings:") +def step_yaml_invalid_mapping(context): + context.yaml_content = context.text.strip("\n") + + +@given("I have a messy value context for rendering") +def step_context_messy_value(context): + context.template_context = {"messy_value": "value1 key2: value2"} + + +@when("I render the malformed YAML content") +def step_render_malformed_yaml(context): + try: + context.malformed_result = context.yaml_engine.load_string( + context.yaml_content, context.template_context + ) + context.malformed_error = None + except Exception as exc: # pragma: no cover - exercised in tests + context.malformed_result = None + context.malformed_error = exc + + +@then("the parser should attempt fixes then raise YAML error") +def step_assert_malformed_error(context): + assert context.malformed_result is None + assert isinstance(context.malformed_error, yaml.YAMLError) + + +@given("I have a deferred YAML template that becomes a list:") +def step_deferred_list_template(context): + context.yaml_content = context.text.strip("\n") + + +@then("a ValueError should be raised for non-mapping deferred templates") +def step_assert_deferred_value_error(context): + assert isinstance(context.deferred_error, ValueError) + + +@given("I have YAML with a placeholder causing parse retry:") +def step_yaml_placeholder_retry(context): + context.yaml_content = context.text.strip("\n") + context.retry_context = None + context.retry_result = None + context.retry_error = None + + +@given("I have a template context for retry parsing") +def step_retry_context(context): + context.retry_context = {"value": "patched", "context": {}} + + +@when("I render YAML with a transient YAML error on first parse") +def step_render_with_transient_error(context): + side_effects = [yaml.YAMLError("transient"), {"result": "recovered"}] + with mock.patch.object(_yaml_module.yaml, "safe_load", side_effect=side_effects): + context.retry_result = context.yaml_engine._render_and_parse( + context.yaml_content, context.retry_context + ) + + +@then("the rendering retry should yield a mapping result") +def step_assert_retry_success(context): + assert context.retry_result == {"result": "recovered"} + + +@when("rendering retry returns non-mapping after YAML error") +def step_render_retry_non_mapping(context): + side_effects = [yaml.YAMLError("still broken"), ["not", "mapping"]] + try: + with mock.patch.object( + _yaml_module.yaml, "safe_load", side_effect=side_effects + ): + context.retry_result = context.yaml_engine._render_and_parse( + context.yaml_content, context.retry_context + ) + except Exception as exc: # pragma: no cover - exercised in tests + context.retry_error = exc + + +@then("a ValueError should be raised for non-mapping retry") +def step_assert_retry_value_error(context): + assert isinstance(context.retry_error, ValueError) + assert "dict" in str(context.retry_error) + + +@given("I have inline YAML with multiple colons on one line:") +def step_inline_multicolon_yaml(context): + context.inline_yaml = context.text.strip("\n") + + +@when("I apply the common YAML fixes") +def step_apply_common_fixes(context): + context.fixed_common = context.yaml_engine._fix_common_yaml_issues( + context.inline_yaml + ) + + +@then("the multi-colon line should be split into nested entries") +def step_assert_common_fixes(context): + expected = "\n".join( + [ + "config:", + " key1:", + " val1", + " key2:", + " val2", + ] + ) + assert context.fixed_common == expected + + +@given("I have synthetic YAML content for postprocess:") +def step_synthetic_postprocess_input(context): + context.synthetic_yaml = context.text.strip("\n") + + +@when("I postprocess using a single-colon synthetic line") +def step_postprocess_synthetic(context): + class FakeLine(str): + def count(self, sub, start=0, end=None): + if sub == ":": + return 1 + return super().count(sub, start, end) + + class FakeContent(str): + def split(self, sep=None, maxsplit=-1): + sep_to_use = "\n" if sep is None else sep + return [FakeLine(part) for part in super().split(sep_to_use, maxsplit)] + + fake_content = FakeContent(context.synthetic_yaml) + context.synthetic_postprocessed = context.yaml_engine._postprocess_rendered_yaml( + fake_content + ) + + +@then("the synthetic postprocess should keep the line intact") +def step_assert_synthetic_postprocess(context): + assert context.synthetic_postprocessed.strip() == "config: alpha beta: gamma" diff --git a/features/yaml_template_engine_coverage.feature b/features/yaml_template_engine_coverage.feature new file mode 100644 index 000000000..18a07a7f6 --- /dev/null +++ b/features/yaml_template_engine_coverage.feature @@ -0,0 +1,175 @@ +Feature: YAML Template Engine coverage + Background: + Given the YAML template engine is initialized for coverage + + Scenario: Postprocess splits combined mapping values + Given I have YAML with specific line splitting pattern: + """ + config: + combined: value1 key2: value2 + test: normal value + """ + When I process postprocessing line splitting + Then specific line splitting should occur + And the lines should be restructured + + Scenario: Rejects non-dict plain YAML without templates + Given I have plain YAML without templates: + """ + - item1 + - item2 + """ + When I load YAML content through the template engine + Then a ValueError should be raised indicating dict requirement + + Scenario: Deferred loading preserves template placeholders + Given the YAML template engine is initialized for coverage + Given I have a YAML string with inline template placeholders: + """ + agent: + name: "{{ name }}" + model: "{{ model }}" + """ + When I load YAML content without context for deferred rendering + Then the deferred template should keep placeholders + And the deferred template should be a mapping with key "agent" + + + Scenario: Deferred loading surfaces YAML errors for invalid templates + Given I have invalid templated YAML for deferred rendering: + """ + agent: + name: {{ name + """ + When I load YAML content without context for deferred rendering + Then a YAML error should be raised for deferred loading + + Scenario: Preprocess inserts indentation hints for for-loops + Given I have a YAML string with for loops requiring preprocessing: + """ + agents: + {% for item in items %} + - name: {{ item }} + {% endfor %} + """ + When I preprocess the YAML content for rendering + Then indentation hints should be present after the loop line + + Scenario: Rendering with colon-heavy values triggers fallback fix + Given I have a templated YAML that renders malformed mapping: + """ + config: {{ value_with_colons }} + """ + And I have a template context with colon rich value + When I render the YAML with context + Then a YAML parsing error should be raised after attempting fixes + + Scenario: Rendering simple template succeeds with merged context + Given I have a YAML string with inline Jinja2 templates: + """ + items: + {% for item in items %} + - {{ item }} + {% endfor %} + context_value: {{ context.existing }} + """ + And I have a template context with list items + When I render the YAML with context + Then the rendered YAML should be parsed into a mapping + And the rendered mapping should include loop results and merged context + + Scenario: Selectattr filter fills defaults + Given I have objects with missing attributes + When I apply the selectattr filter for attribute "score" with default 0 + Then the selected attributes should include defaults + + Scenario: Sum and helper filters format values + Given I have numeric values for the sum filter + When I apply the sum filter and helper formatters + Then the helper filters should format output + + Scenario: Analyze YAML structure skips comments and finds templates + Given I have YAML with mixed content for structure analysis: + """ + # comment + root: + value: static + {{ inline_var }} + {% for x in items %} + - name: {{ x }} + {% endfor %} + """ + When I analyze the YAML structure + Then template blocks and inline templates should be located + + Scenario: Load file wrapper processes templates from disk + Given I have a temporary YAML file with template content: + """ + agent: + name: "{{ name }}" + role: static + """ + When I load the YAML file with context data + Then the file load result should merge templated values + + Scenario: Rendering list template raises mapping ValueError + Given I have a templated YAML that renders to a list root: + """ + {% for item in items %} + - {{ item }} + {% endfor %} + """ + And I have a template context with list items + When I render the YAML expecting a non-mapping error + Then a ValueError should indicate rendered YAML must be a mapping + + Scenario: Malformed rendered YAML triggers fallback fix path + Given I have templated YAML that renders invalid mappings: + """ + config: {{ messy_value }} + """ + And I have a messy value context for rendering + When I render the malformed YAML content + Then the parser should attempt fixes then raise YAML error + + Scenario: Deferred templated list raises mapping ValueError + Given I have a deferred YAML template that becomes a list: + """ + - "{{ value }}" + """ + When I load YAML content without context for deferred rendering + Then a ValueError should be raised for non-mapping deferred templates + + Scenario: Parse retry succeeds after YAML error + Given I have YAML with a placeholder causing parse retry: + """ + result: {{ value }} + """ + And I have a template context for retry parsing + When I render YAML with a transient YAML error on first parse + Then the rendering retry should yield a mapping result + + Scenario: Parse retry raises ValueError when repaired YAML is not a mapping + Given I have YAML with a placeholder causing parse retry: + """ + result: {{ value }} + """ + And I have a template context for retry parsing + When rendering retry returns non-mapping after YAML error + Then a ValueError should be raised for non-mapping retry + + Scenario: Fix common YAML issues splits multiple colons + Given I have inline YAML with multiple colons on one line: + """ + config: key1: val1 key2: val2 + """ + When I apply the common YAML fixes + Then the multi-colon line should be split into nested entries + + Scenario: Synthetic postprocess respects single-colon guard + Given I have synthetic YAML content for postprocess: + """ + config: alpha beta: gamma + """ + When I postprocess using a single-colon synthetic line + Then the synthetic postprocess should keep the line intact diff --git a/implementation_plan.md b/implementation_plan.md index 4600fe595..4af87f118 100644 --- a/implementation_plan.md +++ b/implementation_plan.md @@ -651,6 +651,13 @@ All 10 ADRs have been created in `docs/architecture/decisions/`: - 2025-12-19: Actor persistence primitives verified in codebase: actors table + default index present via `alembic/versions/c3d9b3d0cf3e_add_actors_table.py:1` and SQLAlchemy model carries default/built-in flags at `src/cleveragents/infrastructure/database/models.py:152`; repository enforces built-in/default guards and exposes default setters in `src/cleveragents/infrastructure/database/repositories.py:582` and `src/cleveragents/infrastructure/database/repositories.py:661`; UnitOfWork and DI wire the repository into services at `src/cleveragents/infrastructure/database/unit_of_work.py:170` and `src/cleveragents/application/container.py:85`; ActorService normalization plus default handling lives at `src/cleveragents/application/services/actor_service.py:22`. No gaps found for Stage 7.5 persistence prerequisite; next work is adapter import + CLI flag changes. +**2025-12-25: V2 actor config parsing (unchanged format) + provenance recorded** +- Ported the full v2 YAML config loader into `src/cleveragents/actor/config.py:37-211`, including Jinja2 template handling (via `YAMLTemplateEngine`), placeholder restoration, env-var interpolation with defaults/typing, and explicit error wrapping for malformed YAML; the actor config format is identical to v2—no new formats or routing styles were added. +- Captured v2 reference commit `10ad86df6069423955e42beacbdf7e36dfa779da` for actor parser provenance; future deviations must reconcile against this tag to keep the v2 format intact. +- Added dependency `jinja2>=3.1.0` (project deps) to match the v2 parser requirements. +- Behave coverage added in `features/actor_config_coverage.feature:64-96` with steps in `features/steps/actor_config_steps.py:138-151` validating provider/model inference, unsafe propagation, graph_descriptor keys, option merge behavior, and YAML parse error surfaces (all in v2 format). +- Robot integration added via `robot/helper_actor_config.py:1-29` and `robot/actor_configuration.robot:1-41` to assert provider/model/graph key extraction from v2-format YAML configs end-to-end. + **2025-12-09: Provider streaming normalization + Behave coverage refresh** - `PlanService.generate_plan_streaming` now consumes the new provider iterator contract, normalizes nested `__end__/response` payloads, validates streamed `Change` objects, persists token counts, and still emits the legacy CLI end event for compatibility (`src/cleveragents/application/services/plan_service.py:858`). - `LangChainChatProvider.stream_changes` (plus both mock providers) now yield LangGraph workflow events followed by `{"__end__": {"response": ProviderResponse}}`, guaranteeing a consistent exit shape and structured usage logging (`src/cleveragents/providers/llm/langchain_chat_provider.py:154`, `features/mocks/langchain_mock_provider.py:269`, `features/mocks/mock_ai_provider.py:218`). @@ -4443,12 +4450,12 @@ If you can do all of the above by end of Day 1, you're on track! - [X] Add actor persistence primitives (database schema/migration + repository + DI wiring). - [X] Create `src/cleveragents/actor/` package skeleton (module and `__init__`) to host the ported v2 logic and register with the DI container. - [X] Check out git tag `v2` as a read-only worktree at `./v2` (reference-only, same rule as `./plandex/`; no imports into v3 code). - - [ ] Port the relevant Python from the git `v2` tag into `src/cleveragents/actor/` as first-class v3 code (no `v2` references in code or comments); pin the referenced commit hash in **Phase 2 Notes**; add any new dependencies to `[project.optional-dependencies.actors]`; keep package `__init__` files side-effect free so the actor package controls initialization. - - [ ] Capture the git `v2` commit hash for actor parser provenance and reconcile deviations once the tag is available (add to Phase 2 Notes). - - [ ] Fix – record git `v2` actor commit hash once repository tag access is available (currently unavailable in this workspace). + - [X] Port the relevant Python from the git `v2` tag into `src/cleveragents/actor/` as first-class v3 code (no `v2` references in code or comments); pin the referenced commit hash in **Phase 2 Notes**; add any new dependencies to `[project.optional-dependencies.actors]`; keep package `__init__` files side-effect free so the actor package controls initialization. + - [X] Capture the git `v2` commit hash for actor parser provenance and reconcile deviations once the tag is available (add to Phase 2 Notes). + - [X] Fix – record git `v2` actor commit hash once repository tag access is available (currently unavailable in this workspace). - [X] Port the actor-configuration parser/runner from the git `v2` tag into the actor package so it emits `graph_descriptor`, provider/model requirements, normalized options, and `unsafe`; map its flags to a typed Pydantic config model; bind it to the existing provider registry and `ContextService` so all context/state flows through current lifecycles, inject package-produced initial context variables before graph execution, and ensure all call sites use actor names instead of provider/model pairs. - [X] Recover or define the canonical actor configuration schema/graph descriptor (from the git `v2` tag or a reconstructed spec) to drive the parser, hashing rules, and validation for Stage 7.5. - + - [X] Confirm actor config parser is functionally identical to v2 format (no new formats/routing), with the full v2 loader behavior ported into v3. - [X] Implement an actor registry backed by the config DB with schema: `name`, `config_blob` (canonical JSON/YAML, no file path), `config_hash` (content hash), `graph_descriptor`, `unsafe` (bool), `created_at`, `updated_at`, `default_actor`; generate built-ins from the provider registry (`/`) at startup as read-only rows; enforce `local/` naming for customs; block removal when target is default; expose CRUD via repository + service layer; add migration to create/modify registry tables and the default pointer. - [X] Wire CLI: `actor add --name --config [--unsafe]` (auto-prefix `local/`, compute hash, persist blob, require `--unsafe` when the actor config is marked unsafe), `actor update --name [--config ] [--unsafe|--safe] [--set-default]` (mutually exclusive unsafe/safe; allow default change without new config), `actor remove local/` (error if default; ensure record exists), `actor list` (built-in/custom, default marker, unsafe flag, hash, created/updated timestamps), `actor show ` (unsafe flag, stored hash, graph summary, provider/model requirements, config excerpt). Reject names missing required prefixes and enforce unsafe/safe exclusivity. - [X] Update `chat`/`plan` and any other provider/model entry points to require `--actor` (remove `--model/--provider` flags and help text), honor `default_actor` when flag is omitted, retain git v2 tag's `run` flags (`--context`, `--load-context`, etc.) via `ContextService`, emit warnings at verbosity ≥ warning for unsafe actors (runtime does not require `--unsafe`), and allow built-in or custom actors as default. Ensure selection resolves to provider registry entries with per-model options forwarded and package-produced graph descriptors injected into execution, and reject raw provider/model usage in favor of actor names. @@ -4457,16 +4464,15 @@ If you can do all of the above by end of Day 1, you're on track! - [ ] Port git `v2` tag API/CLI documentation relevant to actors/configuration into docs; scrub `--model/--provider` references and any `v2` naming; update CLI help/man pages and `agents --help` output to describe `--actor`, unsafe semantics, default resolution, and removal guards. - [ ] Delete the temporary `./v2` reference directory once Stage 7.5 actor porting and documentation are complete (final cleanup step). - [ ] Document: - - [ ] Update **Phase 2 Notes**, README, and docs (actor/CLI pages) with actor naming rules (`/` built-ins, `local/` customs), registry storage in config DB (blob + hash + unsafe + graph_descriptor + timestamps + default pointer), default selection behavior, unsafe semantics (`--unsafe`/`--safe` exclusivity), removal guard for defaults, built-in immutability, and context flag parity with v2. - - [ ] Document actor command examples for add/update/set-default/remove/list/show and chat/plan usage, including sample warnings for unsafe actors at verbosity ≥ warning, default resolution examples, failure cases (missing config, missing prefix, attempting to remove default, unsafe flag required), and note that runtime does not require `--unsafe` to execute. + - [ ] Update **Phase 2 Notes**, README, and docs (actor/CLI pages) with actor naming rules (`/` built-ins, `local/` customs), registry storage in config DB (blob + hash + unsafe + graph_descriptor + timestamps + default pointer), default selection behavior, unsafe semantics (`--unsafe`/`--safe` exclusivity), removal guard for defaults, built-in immutability, context flag parity with v2, and the requirement that actor config files remain in the exact v2 format (no new formats/routing). + - [ ] Document actor command examples for add/update/set-default/remove/list/show and chat/plan usage, including sample warnings for unsafe actors at verbosity ≥ warning, default resolution examples, failure cases (missing config, missing prefix, attempting to remove default, unsafe flag required), runtime not requiring `--unsafe`, and explicit statements that actor config files must conform to the v2 format without extensions or alternative schemas. - [ ] Update architecture references/ADRs (ADR-008/ADR-011 or addenda) to capture actor registry boundaries, actor package responsibilities, dependency on provider registry + `ContextService`, default pointer storage, built-in immutability, and removal of `--model/--provider` in favor of `--actor`. - [ ] Include migrated API documentation from git's v2 tag describing actor configuration fields, supported options, graph descriptor semantics, unsafe detection rules, and how actor package outputs map to provider/model selection and LangGraph invocation. - [ ] Tests: + - [X] Add v2-format actor config parsing coverage (Behave + Robot) for YAML inference of provider/model/graph/options (format unchanged between v2 and v3). - [ ] Port git v2's tag unit coverage into Behave features covering actor configuration parsing (valid/invalid blobs, option normalization), actor package outputs (graph_descriptor, provider/model requirements), unsafe detection, registry CRUD (hash stability, default guard, removal requiring `local/`), built-in enumeration, default selection, warning emission at verbosity ≥ warning, and chat/plan flows with `--actor` plus context flags. - [ ] Migrate existing provider/model Behave and Robot suites (plan CLI, plan_service/provider_registry overrides, streaming/auto-debug flows) to actor-only flags and expectations so provider/model paths are removed without coverage regressions. - - [ ] Port git v2 tag's integration/e2e coverage into Robot suites for actor add/update/remove/list/show, default-actor deletion guard, unsafe warning display, chat/plan end-to-end with custom and built-in actors, and persistence of context across invocations with `--context`/`--load-context`; include DB persistence checks for blob/hash/unsafe/default fields. - - [ ] Add Behave/Robot tests ensuring config DB stores canonical blobs (no path reliance), hash only changes on update, `--unsafe`/`--safe` mutual exclusivity enforcement, default fallback when `--actor` is omitted, built-in immutability, provider-model option forwarding, and warning emission without requiring runtime `--unsafe`. - [ ] Update nox sessions to include new actor Behave/Robot suites; replace/remove `--model/--provider` coverage with actor-based tests; ensure coverage stays ≥85% and pyright remains clean; verify docs build for ported API references and CLI help updates. - [ ] Stage 8: Async Infrastructure diff --git a/pyproject.toml b/pyproject.toml index 6e517b2d1..7374add61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "structlog>=24.0.0", # Configuration "pyyaml>=6.0.2", + "jinja2>=3.1.0", "python-dotenv>=1.0.0", # Phase 2: Database & ORM "sqlalchemy>=2.0.0", diff --git a/robot/actor_configuration.robot b/robot/actor_configuration.robot new file mode 100644 index 000000000..c756ea60c --- /dev/null +++ b/robot/actor_configuration.robot @@ -0,0 +1,39 @@ +*** Settings *** +Library OperatingSystem +Library Collections +Library Process + +*** Test Cases *** +V2 Actor Config Produces Provider And Graph Descriptor + ${config}= Set Variable ${OUTPUT DIR}/v2_actor.yaml + ${content}= Catenate SEPARATOR=\n + ... cleveragents: + ... ${SPACE*2}default_router: main_router + ... agents: + ... ${SPACE*2}paper_writer: + ... ${SPACE*4}type: llm + ... ${SPACE*4}config: + ... ${SPACE*6}provider: openai + ... ${SPACE*6}model: gpt-4 + ... ${SPACE*6}unsafe: true + ... ${SPACE*6}options: + ... ${SPACE*8}temperature: 0.5 + ... routes: + ... ${SPACE*2}main_router: + ... ${SPACE*4}type: stream + ... ${SPACE*4}operators: + ... ${SPACE*6}- type: map + ... ${SPACE*8}params: + ... ${SPACE*10}agent: paper_writer + ... ${SPACE*4}publications: + ... ${SPACE*6}- __output__ + Create File ${config} ${content} + ${result}= Run Process ${CURDIR}/../.nox/integration_tests-3-13/bin/python robot/helper_actor_config.py ${config} stdout=PIPE stderr=PIPE + Should Be Equal As Integers ${result.rc} 0 + ${payload}= Evaluate __import__('json').loads('''${result.stdout.strip()}''') + Should Be Equal ${payload['provider']} openai + Should Be Equal ${payload['model']} gpt-4 + Should Be True ${payload['unsafe']} + List Should Contain Value ${payload['graph_keys']} agents + List Should Contain Value ${payload['graph_keys']} routes + Should Be Equal As Numbers ${payload['options']['temperature']} 0.5 diff --git a/robot/helper_actor_config.py b/robot/helper_actor_config.py new file mode 100644 index 000000000..3531c0e93 --- /dev/null +++ b/robot/helper_actor_config.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from cleveragents.actor.config import ActorConfiguration + + +def main() -> None: + if len(sys.argv) < 2: + raise SystemExit("config path required") + + config_path = Path(sys.argv[1]) + config = ActorConfiguration.from_file(path=config_path) + payload = { + "provider": config.provider, + "model": config.model, + "unsafe": config.unsafe, + "graph_keys": sorted(config.graph_descriptor.keys()) + if config.graph_descriptor + else [], + "options": config.options, + } + print(json.dumps(payload)) + + +if __name__ == "__main__": + main() diff --git a/src/cleveragents/actor/config.py b/src/cleveragents/actor/config.py index f54e8c0cf..d19f823a4 100644 --- a/src/cleveragents/actor/config.py +++ b/src/cleveragents/actor/config.py @@ -1,13 +1,15 @@ -"""Actor configuration models and parsing utilities.""" - from __future__ import annotations import json +import os +import re from pathlib import Path from typing import Any, cast from pydantic import BaseModel, Field +from cleveragents.actor.yaml_template_engine import YAMLTemplateEngine + try: # Optional dependency; present in dev/test extras import yaml except Exception: # pragma: no cover - defensive optional import @@ -32,7 +34,7 @@ class ActorConfiguration(BaseModel): @staticmethod def load_blob_from_file(config_path: Path) -> dict[str, Any]: - """Load a configuration blob from a JSON or YAML file.""" + """Load a configuration blob from a JSON or YAML file (v2 parity).""" path = config_path.expanduser() if not path.exists(): @@ -41,20 +43,135 @@ class ActorConfiguration(BaseModel): text = path.read_text() try: data = json.loads(text) - except json.JSONDecodeError as exc: - if yaml is None: - raise ValueError("PyYAML is required for YAML actor configs") from exc - try: - data = yaml.safe_load(text) - except Exception as yaml_exc: # pragma: no cover - defensive - raise ValueError(f"Failed to parse config: {yaml_exc}") from yaml_exc + except json.JSONDecodeError: + data = ActorConfiguration._load_v2_yaml_content(text) + else: + if data is None: + return {} + if not isinstance(data, dict): + raise ValueError("Config must be a JSON or YAML object") + return cast(dict[str, Any], data) - if data is None: - return {} - if not isinstance(data, dict): + return data + + @staticmethod + def _load_v2_yaml_content(text: str) -> dict[str, Any]: + if yaml is None: + raise ValueError("PyYAML is required for YAML actor configs") + + raw: Any + try: + if "{%" in text or "{{" in text: + protected = re.sub(r"{{", "<<>>", text) + protected = re.sub(r"}}", "<<>>", protected) + protected = re.sub(r"{%", "<<>>", protected) + protected = re.sub(r"%}", "<<>>", protected) + + engine = YAMLTemplateEngine() + minimal_context: dict[str, dict[str, Any]] = { + "context": { + "paper_details": { + "topic": "", + "length": "", + "audience": "", + "publication": "", + "format": "", + "other": "", + }, + "brainstorming_summary": "", + "vetting_sources": list[str](), + "table_of_contents": "", + "deep_research_sources": "", + "current_section_to_write": "", + "paper_content": dict[str, Any](), + "final_paper_text": "", + "proofread_paper": "", + } + } + raw = engine.load_string(protected, context=minimal_context) + raw = ActorConfiguration._restore_template_syntax(raw) + else: + raw = yaml.safe_load(text) + except Exception as exc: # pragma: no cover - defensive + raise ValueError(f"Failed to parse config: {exc}") from exc + + if raw is None: + raw = {} + if not isinstance(raw, dict): raise ValueError("Config must be a JSON or YAML object") - return cast(dict[str, Any], data) + interpolated = ActorConfiguration._interpolate_env_vars(raw) + if not isinstance(interpolated, dict): + raise ValueError("Config must be a JSON or YAML object") + return cast(dict[str, Any], interpolated) + + @staticmethod + def _restore_template_syntax(config: Any) -> Any: + if isinstance(config, dict): + config_dict = cast(dict[str, Any], config) + restored: dict[str, Any] = {} + for key, value in config_dict.items(): + if key == "system_prompt" and isinstance(value, str): + updated = value.replace("<<>>", "{{").replace( + "<<>>", "}}" + ) + updated = updated.replace("<<>>", "{%") + updated = updated.replace("<<>>", "%}") + restored[key] = updated + else: + restored[key] = ActorConfiguration._restore_template_syntax(value) + return restored + if isinstance(config, list): + config_list = cast(list[Any], config) + return [ + ActorConfiguration._restore_template_syntax(item) + for item in config_list + ] + return config + + @staticmethod + def _interpolate_env_vars(config: Any) -> Any: + if isinstance(config, dict): + config_dict = cast(dict[str, Any], config) + return { + k: ActorConfiguration._interpolate_env_vars(v) + for k, v in config_dict.items() + } + if isinstance(config, list): + config_list = cast(list[Any], config) + return [ActorConfiguration._interpolate_env_vars(i) for i in config_list] + if isinstance(config, str): + env_var_pattern = r"\${([A-Za-z0-9_]+)(?::([^}]*))?\}" + + def replace_env_var(match: re.Match[str]) -> str: + env_var = match.group(1) + default_value = match.group(2) if match.group(2) is not None else None + + env_value = os.environ.get(env_var) + if env_value is None: + if default_value is not None: + if default_value.lower() in {"true", "false"}: + return str(default_value.lower() == "true") + if default_value.lstrip("-").isdigit(): + return default_value + return default_value + raise ValueError(f"Environment variable '{env_var}' is not set") + return env_value + + substituted = re.sub(env_var_pattern, replace_env_var, config) + + lowered = substituted.lower() + if lowered in {"true", "false"}: + return lowered == "true" + if substituted.lstrip("-").isdigit(): + return int(substituted) + if ( + substituted.lstrip("-").replace(".", "", 1).isdigit() + and substituted.count(".") == 1 + ): + return float(substituted) + return substituted + return config @classmethod def from_file( @@ -96,12 +213,18 @@ class ActorConfiguration(BaseModel): data: dict[str, Any] = dict(blob) if isinstance(blob, dict) else {} + v2_provider, v2_model, v2_graph, v2_unsafe = cls._extract_v2_actor(data) + v2_options = cls._extract_v2_options(data) + resolved_provider = ( - provider or data.get("provider") or data.get("provider_type") + provider or data.get("provider") or data.get("provider_type") or v2_provider ) - resolved_model = model or data.get("model") or data.get("model_id") + resolved_model = model or data.get("model") or data.get("model_id") or v2_model resolved_graph = ( - graph_descriptor or data.get("graph_descriptor") or data.get("graph") + graph_descriptor + or data.get("graph_descriptor") + or data.get("graph") + or v2_graph ) options_raw = data.get("options") @@ -111,12 +234,14 @@ class ActorConfiguration(BaseModel): merged_options: dict[str, Any] = {} if default_options: merged_options.update(default_options) + if v2_options: + merged_options.update(v2_options) if options_value: merged_options.update(options_value) if option_overrides: merged_options.update(option_overrides) - resolved_unsafe = bool(data.get("unsafe", False)) or unsafe + resolved_unsafe = bool(data.get("unsafe", False)) or v2_unsafe or unsafe if not resolved_provider: raise ValueError("provider is required") @@ -138,3 +263,65 @@ class ActorConfiguration(BaseModel): options=merged_options, unsafe=resolved_unsafe, ) + + @staticmethod + def _extract_v2_actor( + data: dict[str, Any], + ) -> tuple[str | None, str | None, dict[str, Any] | None, bool]: + """Derive provider/model/graph from the v2 YAML actor config format.""" + + agents_obj = data.get("agents") + if isinstance(agents_obj, dict) and agents_obj: + agents_dict = cast(dict[str, Any], agents_obj) + for first_agent, first_entry in agents_dict.items(): + if isinstance(first_entry, dict): + entry_dict = cast(dict[str, Any], first_entry) + config_block = entry_dict.get("config") + if isinstance(config_block, dict): + config_dict = cast(dict[str, Any], config_block) + provider_value = config_dict.get("provider") or config_dict.get( + "provider_type" + ) + model_value = config_dict.get("model") or config_dict.get( + "model_id" + ) + unsafe_flag = bool(config_dict.get("unsafe", False)) + descriptor: dict[str, Any] = { + "agent": first_agent, + "agents": agents_dict, + } + for key in ( + "routes", + "merges", + "templates", + "cleveragents", + ): + if key in data and data[key] is not None: + descriptor[key] = data[key] + return ( + str(provider_value) if provider_value else None, + str(model_value) if model_value else None, + descriptor, + unsafe_flag, + ) + break + return None, None, None, False + + @staticmethod + def _extract_v2_options(data: dict[str, Any]) -> dict[str, Any] | None: + """Extract option defaults from the v2 actor config format.""" + + agents_obj = data.get("agents") + if isinstance(agents_obj, dict) and agents_obj: + agents_dict = cast(dict[str, Any], agents_obj) + for _, first_entry in agents_dict.items(): + if isinstance(first_entry, dict): + entry_dict = cast(dict[str, Any], first_entry) + config_block = entry_dict.get("config") + if isinstance(config_block, dict): + config_dict = cast(dict[str, Any], config_block) + options = config_dict.get("options") + if isinstance(options, dict): + return cast(dict[str, Any], options) + break + return None diff --git a/src/cleveragents/actor/yaml_template_engine.py b/src/cleveragents/actor/yaml_template_engine.py new file mode 100644 index 000000000..cf4f7ff0f --- /dev/null +++ b/src/cleveragents/actor/yaml_template_engine.py @@ -0,0 +1,242 @@ +"""YAML Template Engine with inline Jinja2 support (ported from v2).""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Callable, cast + +import yaml +from jinja2 import Environment + +logger = logging.getLogger(__name__) + + +class YAMLTemplateEngine: + """Complete solution for inline Jinja2 templates in YAML (v2 parity).""" + + def __init__(self) -> None: + # Configure Jinja2 for YAML-friendly output + self.env = Environment( + block_start_string="{%", + block_end_string="%}", + variable_start_string="{{", + variable_end_string="}}", + comment_start_string="{#", + comment_end_string="#}", + trim_blocks=False, + lstrip_blocks=False, + keep_trailing_newline=True, + ) + + # Add custom filters for YAML + self.env.filters["yaml"] = self._yaml_filter + self.env.filters["indent"] = self._indent_filter + self.env.filters["sum"] = self._sum_filter + self.env.filters["selectattr"] = cast( + Callable[..., Any], self._selectattr_filter + ) + + def load_file( + self, file_path: Path, context: dict[str, Any] | None = None + ) -> dict[str, Any]: + with open(file_path, encoding="utf-8") as f: + content = f.read() + return self.load_string(content, context) + + def load_string( + self, yaml_content: str, context: dict[str, Any] | None = None + ) -> dict[str, Any]: + if "{%" not in yaml_content and "{{" not in yaml_content: + loaded = yaml.safe_load(yaml_content) + if not isinstance(loaded, dict): + raise ValueError( + f"Expected YAML to load as dict, got {type(loaded).__name__}" + ) + return cast(dict[str, Any], loaded) + + if context is not None: + return self._render_and_parse(yaml_content, context) + return self._prepare_for_deferred_rendering(yaml_content) + + def _render_and_parse( + self, yaml_content: str, context: dict[str, Any] + ) -> dict[str, Any]: + processed_content = self._preprocess_for_rendering(yaml_content) + full_context = self._create_render_context(context) + template = self.env.from_string(processed_content) + rendered = template.render(**full_context) + fixed_yaml = self._postprocess_rendered_yaml(rendered) + + try: + parsed = yaml.safe_load(fixed_yaml) + if not isinstance(parsed, dict): + raise ValueError( + f"Expected YAML to parse as dict, got {type(parsed).__name__}" + ) + return cast(dict[str, Any], parsed) + except yaml.YAMLError as exc: + logger.error("Failed to parse rendered YAML: %s", exc) + logger.debug("Rendered YAML:\n%s", fixed_yaml) + fixed_yaml = self._fix_common_yaml_issues(fixed_yaml) + parsed = yaml.safe_load(fixed_yaml) + if not isinstance(parsed, dict): + raise ValueError( + f"Expected YAML to parse as dict, got {type(parsed).__name__}" + ) from exc + return cast(dict[str, Any], parsed) + + def _preprocess_for_rendering(self, content: str) -> str: + lines = content.split("\n") + processed_lines: list[str] = [] + for line in lines: + if "{%" in line and "for" in line: + indent = len(line) - len(line.lstrip()) + processed_lines.append(line) + processed_lines.append(f"{' ' * indent}{{# indent: {indent} #}}") + else: + processed_lines.append(line) + return "\n".join(processed_lines) + + def _postprocess_rendered_yaml(self, content: str) -> str: + lines = content.split("\n") + fixed_lines: list[str] = [] + for i, line in enumerate(lines): + if 0 < i < len(lines) - 1 and line.strip() == "": + next_line = lines[i + 1] if i + 1 < len(lines) else "" + if next_line and not next_line.startswith(" "): + continue + + if ":" in line and line.count(":") == 1: + parts = line.split(":", 1) + if len(parts) == 2: + key_part = parts[0] + value_part = parts[1].strip() + if value_part and " " in value_part and ":" not in value_part: + value_words = value_part.split() + if len(value_words) > 1 and any( + ":" in w for w in value_words[1:] + ): + fixed_lines.append(f"{key_part}: {value_words[0]}") + remaining = " ".join(value_words[1:]) + indent = len(key_part) - len(key_part.lstrip()) + fixed_lines.append(f"{' ' * indent}{remaining}") + continue + fixed_lines.append(line) + return "\n".join(fixed_lines) + + def _fix_common_yaml_issues(self, content: str) -> str: + lines = content.split("\n") + fixed_lines: list[str] = [] + for line in lines: + if line.count(":") > 1: + parts = line.split(":", 1) + if len(parts) == 2: + indent = len(parts[0]) - len(parts[0].lstrip()) + fixed_lines.append(f"{parts[0]}:") + remaining = parts[1].strip() + if remaining: + tokens = remaining.split() + current_line = "" + for token in tokens: + if ":" in token and token.endswith(":"): + if current_line: + fixed_lines.append( + f"{' ' * (indent + 2)}{current_line}" + ) + fixed_lines.append(f"{' ' * (indent + 2)}{token}") + current_line = "" + else: + current_line += f" {token}" if current_line else token + if current_line: + fixed_lines.append(f"{' ' * (indent + 2)}{current_line}") + continue + fixed_lines.append(line) + return "\n".join(fixed_lines) + + def _prepare_for_deferred_rendering(self, yaml_content: str) -> dict[str, Any]: + return self._simple_template_extraction(yaml_content) + + def _analyze_yaml_structure(self, content: str) -> dict[str, Any]: + lines = content.split("\n") + structure: dict[str, Any] = { + "template_blocks": [], + "inline_templates": [], + "hierarchy": [], + } + current_path: list[tuple[str, int]] = [] + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + while current_path and indent <= current_path[-1][1]: + current_path.pop() + if ":" in line: + key = line.split(":", 1)[0].strip() + current_path.append((key, indent)) + structure["hierarchy"].append( + { + "line": i, + "key": key, + "indent": indent, + "path": [p[0] for p in current_path], + } + ) + if "{%" in line: + structure["template_blocks"].append(i) + if "{{" in line: + structure["inline_templates"].append(i) + return structure + + def _simple_template_extraction(self, content: str) -> dict[str, Any]: + rendered = content + try: + parsed = yaml.safe_load(rendered) + if not isinstance(parsed, dict): + raise ValueError( + f"Expected YAML to load as dict, got {type(parsed).__name__}" + ) + return cast(dict[str, Any], parsed) + except yaml.YAMLError as exc: + logger.error("Failed to parse YAML with templates: %s", exc) + logger.debug("YAML content with templates:\n%s", rendered) + raise + + @staticmethod + def _yaml_filter(value: Any) -> str: + return yaml.dump(value, default_flow_style=False).strip() + + @staticmethod + def _indent_filter(value: Any, spaces: int = 2) -> str: + return "\n".join(" " * spaces + line for line in str(value).split("\n")) + + @staticmethod + def _sum_filter(value: Any) -> Any: + try: + return sum(value) + except Exception as exc: # pragma: no cover - defensive + raise ValueError(f"Cannot sum value: {value}") from exc + + @staticmethod + def _selectattr_filter( + sequence: list[Any], attr: str, default: Any = None + ) -> list[Any]: + results: list[Any] = [] + for item in sequence: + if hasattr(item, attr): + results.append(getattr(item, attr)) + else: + results.append(default) + return results + + @staticmethod + def _create_render_context(context: dict[str, Any]) -> dict[str, Any]: + nested_context = context.get("context") + nested_mapping = ( + cast(dict[str, Any], nested_context) + if isinstance(nested_context, dict) + else {} + ) + base_context: dict[str, Any] = {"context": nested_mapping.copy()} + return base_context | context