From f1bb0bf075c58d8fac78620b6a8a88a21059da82 Mon Sep 17 00:00:00 2001 From: CleverThis Date: Tue, 14 Apr 2026 12:49:38 +0000 Subject: [PATCH 1/2] fix(application): Remove error suppression in reactive_registry_adapter.py Remove two try...except Exception: blocks that were silently suppressing errors in register_registry_agents(), violating CONTRIBUTING.md fail-fast policy. Changes: - Remove try/except around actor_registry.list_actors() call; exceptions now propagate to the caller instead of silently returning - Remove try/except around route_bridge.agents refresh; exceptions now propagate instead of silently resetting to empty dict - Update docstring to document the fail-fast propagation behaviour - Update Behave scenarios to verify exceptions propagate correctly: * RuntimeError from list_actors() propagates * AttributeError from actors without .name attribute propagates * TypeError from None actors list propagates Closes #9060 --- features/consolidated_routing.feature | 21 ++++----- .../steps/reactive_registry_adapter_steps.py | 45 +++++++++++++++++++ .../application/reactive_registry_adapter.py | 13 +++--- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/features/consolidated_routing.feature b/features/consolidated_routing.feature index e0b8d447c..49febb07d 100644 --- a/features/consolidated_routing.feature +++ b/features/consolidated_routing.feature @@ -35,12 +35,11 @@ Feature: Consolidated Routing # Feature: Reactive registry adapter coverage # ============================================================ - Scenario: Registry list failure is ignored + Scenario: Registry list failure propagates to caller Given a reactive stream router and route bridge And an actor registry that raises on list - When I register registry actors with the adapter - Then no agents are registered - And the route bridge agents remain unchanged + When I attempt to register registry actors with the adapter + Then a RuntimeError is raised Scenario: Registers only actors with processing methods @@ -51,20 +50,18 @@ Feature: Consolidated Routing And the route bridge agents cache includes all listed actors - Scenario: Route bridge cache resets when refresh fails + Scenario: Route bridge refresh failure propagates to caller Given a reactive stream router and route bridge And an actor registry that returns actors without names - When I register registry actors with the adapter - Then the route bridge agents cache is cleared - And no agents are registered + When I attempt to register registry actors with the adapter + Then an AttributeError is raised - Scenario: Route bridge cache resets when registry returns none + Scenario: Route bridge refresh failure propagates when registry returns none Given a reactive stream router and route bridge And an actor registry that returns none - When I register registry actors with the adapter - Then the route bridge agents cache is cleared - And no agents are registered + When I attempt to register registry actors with the adapter + Then a TypeError is raised # ============================================================ diff --git a/features/steps/reactive_registry_adapter_steps.py b/features/steps/reactive_registry_adapter_steps.py index 480baab22..e7cc813fa 100644 --- a/features/steps/reactive_registry_adapter_steps.py +++ b/features/steps/reactive_registry_adapter_steps.py @@ -61,6 +61,18 @@ def step_register_actors(context): ) +@when("I attempt to register registry actors with the adapter") +def step_attempt_register_actors(context): + """Attempt registration, capturing any exception that propagates.""" + context.raised_exception = None + try: + register_registry_agents( + context.stream_router, context.route_bridge, context.actor_registry + ) + except Exception as exc: + context.raised_exception = exc + + @then("no agents are registered") def step_assert_no_agents(context): assert context.stream_router.agents == {} @@ -88,3 +100,36 @@ def step_assert_bridge_cache(context): @then("the route bridge agents cache is cleared") def step_assert_bridge_cleared(context): assert context.route_bridge.agents == {} + + +@then("a RuntimeError is raised") +def step_assert_runtime_error(context): + assert context.raised_exception is not None, ( + "Expected RuntimeError but no exception was raised" + ) + assert isinstance(context.raised_exception, RuntimeError), ( + f"Expected RuntimeError but got {type(context.raised_exception).__name__}: " + f"{context.raised_exception}" + ) + + +@then("an AttributeError is raised") +def step_assert_attribute_error(context): + assert context.raised_exception is not None, ( + "Expected AttributeError but no exception was raised" + ) + assert isinstance(context.raised_exception, AttributeError), ( + f"Expected AttributeError but got {type(context.raised_exception).__name__}: " + f"{context.raised_exception}" + ) + + +@then("a TypeError is raised") +def step_assert_type_error(context): + assert context.raised_exception is not None, ( + "Expected TypeError but no exception was raised" + ) + assert isinstance(context.raised_exception, TypeError), ( + f"Expected TypeError but got {type(context.raised_exception).__name__}: " + f"{context.raised_exception}" + ) diff --git a/src/cleveragents/application/reactive_registry_adapter.py b/src/cleveragents/application/reactive_registry_adapter.py index 1248a4208..cdaaa419e 100644 --- a/src/cleveragents/application/reactive_registry_adapter.py +++ b/src/cleveragents/application/reactive_registry_adapter.py @@ -16,11 +16,11 @@ def register_registry_agents( This expects actor_registry to expose list_actors() and get_actor(name). Each actor is injected as a callable via process_message_sync if present; otherwise skips. + + Exceptions from actor_registry.list_actors() or route_bridge refresh are + allowed to propagate to the caller per CONTRIBUTING.md fail-fast policy. """ - try: - actors = actor_registry.list_actors() # type: ignore[attr-defined] - except Exception: - return + actors = actor_registry.list_actors() # type: ignore[attr-defined] for actor in actors or []: # Prefer synchronous processing if available @@ -29,7 +29,4 @@ def register_registry_agents( # RouteBridge caches agents as a dict; refresh it if hasattr(route_bridge, "agents"): - try: - route_bridge.agents = {a.name: a for a in actors} - except Exception: - route_bridge.agents = {} + route_bridge.agents = {a.name: a for a in actors} -- 2.52.0 From 790eb6f001e837a81c1d76fec24ad590aceaed7e Mon Sep 17 00:00:00 2001 From: CleverThis Date: Tue, 14 Apr 2026 12:54:19 +0000 Subject: [PATCH 2/2] test(data): introduce dynamic data generation and externalize test data in Behave and Robot Framework suites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added Faker dependency to pyproject.toml for dynamic test data generation - Created features/test_data_factory.py with TestDataGenerator and ContextFragmentFactory classes for Behave tests - Created robot/helper_test_data_factory.py with RobotTestDataGenerator and factory classes for Robot Framework tests - Created features/test_data_loader.py to load externalized test data from JSON files - Created features/fixtures/test_data_samples.json with realistic test data samples - Updated robot/helper_acms_fusion.py to use dynamic test data generation instead of hardcoded values like "alpha" and "beta" - All quality gates passing: lint, typecheck, unit tests, integration tests, coverage ≥ 97% ISSUES CLOSED: #9048 --- features/fixtures/test_data_samples.json | 132 ++++++++ features/test_data_factory.py | 236 ++++++++++++++ features/test_data_loader.py | 92 ++++++ pyproject.toml | 1 + robot/helper_acms_fusion.py | 131 ++++---- robot/helper_test_data_factory.py | 374 +++++++++++++++++++++++ 6 files changed, 904 insertions(+), 62 deletions(-) create mode 100644 features/fixtures/test_data_samples.json create mode 100644 features/test_data_factory.py create mode 100644 features/test_data_loader.py create mode 100644 robot/helper_test_data_factory.py diff --git a/features/fixtures/test_data_samples.json b/features/fixtures/test_data_samples.json new file mode 100644 index 000000000..05d9cc7ef --- /dev/null +++ b/features/fixtures/test_data_samples.json @@ -0,0 +1,132 @@ +{ + "project_names": [ + "data-pipeline", + "ml-framework", + "api-gateway", + "auth-service", + "cache-layer", + "event-stream", + "config-manager", + "logging-system", + "monitoring-tool", + "deployment-engine" + ], + "skill_names": [ + "text_analysis", + "code_generation", + "data_validation", + "error_handling", + "performance_tuning", + "security_audit", + "documentation_gen", + "test_generation", + "refactoring_assist", + "debugging_help" + ], + "tool_names": [ + "file-reader", + "api-caller", + "database-query", + "cache-store", + "message-queue", + "webhook-sender", + "log-aggregator", + "metric-collector", + "config-loader", + "secret-manager" + ], + "python_code_samples": [ + "def hello_world():\n return 'Hello, World!'", + "result = sum([1, 2, 3, 4, 5])", + "data = {'name': 'Alice', 'age': 30}", + "for i in range(10):\n print(i)", + "import json\ndata = json.loads('{}')", + "class Calculator:\n def add(self, a, b):\n return a + b", + "async def fetch_data():\n return await get_response()", + "try:\n result = risky_operation()\nexcept Exception as e:\n print(e)", + "list_comp = [x * 2 for x in range(5)]", + "lambda_func = lambda x: x ** 2" + ], + "file_paths": [ + "src/core/models.py", + "src/services/auth.py", + "src/handlers/api.py", + "src/utils/helpers.py", + "src/config/settings.py", + "tests/unit/test_models.py", + "tests/integration/test_api.py", + "docs/api_reference.md", + "scripts/migrate_db.py", + "examples/basic_usage.py" + ], + "uko_node_uris": [ + "p://src/models.py", + "p://src/services/auth.py", + "p://src/handlers/api.py", + "p://src/utils/helpers.py", + "p://src/config/settings.py", + "p://tests/unit/test_models.py", + "p://tests/integration/test_api.py", + "p://docs/api_reference.md", + "p://scripts/migrate_db.py", + "p://examples/basic_usage.py" + ], + "resource_uris": [ + "test://database-primary", + "test://cache-redis", + "test://message-queue", + "test://api-gateway", + "test://auth-service", + "test://logging-system", + "test://monitoring-tool", + "test://config-store", + "test://secret-vault", + "test://deployment-engine" + ], + "prose_samples": [ + "This function implements a robust error handling mechanism.", + "The API endpoint validates all incoming requests before processing.", + "Database queries are optimized for performance and scalability.", + "The authentication system uses industry-standard security practices.", + "Configuration management supports multiple environments seamlessly.", + "Logging is comprehensive and includes detailed diagnostic information.", + "Monitoring tools provide real-time visibility into system health.", + "The caching layer reduces database load significantly.", + "Message queues ensure reliable asynchronous communication.", + "Deployment automation minimizes manual intervention and errors." + ], + "edge_case_names": [ + "a", + "my-project", + "my_project", + "my project", + "project123", + "123project", + "My Project Name", + "UPPERCASE", + "lowercase", + "MixedCase" + ], + "invalid_special_chars": [ + "@", + "/", + "!", + "#", + "%", + "^", + "&", + "*", + "+", + "=", + "<", + ">", + "?", + ".", + ":", + ";", + "|", + "\\", + "`", + "~" + ] +} diff --git a/features/test_data_factory.py b/features/test_data_factory.py new file mode 100644 index 000000000..e7c6a20d8 --- /dev/null +++ b/features/test_data_factory.py @@ -0,0 +1,236 @@ +"""Test data factory for Behave tests using Faker for realistic data generation. + +This module provides factory functions and classes for generating realistic test data +for Behave scenarios. It uses Faker to generate dynamic, varied test data instead of +hardcoded values, improving test realism and maintainability. + +Usage: + from features.test_data_factory import ContextFragmentFactory, TestDataGenerator + + # Generate a single context fragment with realistic data + fragment = ContextFragmentFactory.create() + + # Generate multiple fragments with varied data + fragments = ContextFragmentFactory.create_batch(5) + + # Generate realistic project names + project_name = TestDataGenerator.project_name() + + # Generate realistic Python code snippets + code = TestDataGenerator.python_code() +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +from faker import Faker + +# Add src to path for imports +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 + ContextBudget, + ContextFragment, + FragmentProvenance, +) + +# Initialize Faker for consistent, reproducible test data +_faker = Faker() + + +class TestDataGenerator: + """Generate realistic test data for various domain objects.""" + + @staticmethod + def project_name() -> str: + """Generate a realistic project name.""" + return _faker.word() + "-" + _faker.word() + + @staticmethod + def skill_name() -> str: + """Generate a realistic skill name.""" + return _faker.word() + "_" + _faker.word() + + @staticmethod + def tool_name() -> str: + """Generate a realistic tool name.""" + return _faker.word() + "-tool" + + @staticmethod + def actor_name() -> str: + """Generate a realistic actor name.""" + return _faker.first_name() + _faker.last_name() + + @staticmethod + def python_code() -> str: + """Generate a realistic Python code snippet.""" + var_name = _faker.word() + value = _faker.random_int(min=1, max=1000) + return f"{var_name} = {value}" + + @staticmethod + def python_prose() -> str: + """Generate realistic Python-like prose.""" + return _faker.sentence(nb_words=10) + + @staticmethod + def file_path() -> str: + """Generate a realistic file path.""" + return f"src/{_faker.word()}/{_faker.word()}.py" + + @staticmethod + def uko_node_uri() -> str: + """Generate a realistic UKO node URI.""" + return f"p://{_faker.word()}/{_faker.word()}.py" + + @staticmethod + def resource_uri() -> str: + """Generate a realistic resource URI.""" + return f"test://{_faker.word()}-{_faker.word()}" + + @staticmethod + def namespaced_name() -> tuple[str, str]: + """Generate a realistic namespaced name (namespace, name).""" + namespace = _faker.word() + name = _faker.word() + "-" + _faker.word() + return (namespace, name) + + @staticmethod + def ulid() -> str: + """Generate a valid ULID-like string (26 chars, base32).""" + # Generate a valid Crockford base32 ULID (26 characters) + # Using only valid characters: 0-9, A-Z (excluding I, L, O, U) + valid_chars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + return "".join(_faker.random.choice(valid_chars) for _ in range(26)) + + @staticmethod + def relevance_score() -> float: + """Generate a realistic relevance score (0.0 to 1.0).""" + return round(_faker.pyfloat(min_value=0.0, max_value=1.0, right_digits=2), 2) + + @staticmethod + def token_count() -> int: + """Generate a realistic token count.""" + return _faker.random_int(min=10, max=500) + + @staticmethod + def detail_depth() -> int: + """Generate a realistic detail depth level.""" + return _faker.random_int(min=1, max=10) + + +class ContextFragmentFactory: + """Factory for creating realistic ContextFragment test objects.""" + + @staticmethod + def create(**overrides: Any) -> ContextFragment: + """Create a single ContextFragment with realistic default data. + + Args: + **overrides: Override any default values + + Returns: + A ContextFragment instance with realistic data + """ + defaults = { + "uko_node": TestDataGenerator.uko_node_uri(), + "content": TestDataGenerator.python_code(), + "token_count": TestDataGenerator.token_count(), + "relevance_score": TestDataGenerator.relevance_score(), + "detail_depth": TestDataGenerator.detail_depth(), + "provenance": FragmentProvenance( + resource_uri=TestDataGenerator.resource_uri() + ), + } + defaults.update(overrides) + return ContextFragment(**defaults) + + @staticmethod + def create_batch(count: int, **overrides: Any) -> list[ContextFragment]: + """Create multiple ContextFragments with varied realistic data. + + Args: + count: Number of fragments to create + **overrides: Override any default values (applied to all) + + Returns: + A list of ContextFragment instances + """ + return [ContextFragmentFactory.create(**overrides) for _ in range(count)] + + @staticmethod + def create_with_content(content: str, **overrides: Any) -> ContextFragment: + """Create a ContextFragment with specific content. + + Args: + content: The content to use + **overrides: Override any other default values + + Returns: + A ContextFragment instance with the specified content + """ + overrides["content"] = content + return ContextFragmentFactory.create(**overrides) + + @staticmethod + def create_duplicate_pair( + content: str = "duplicate_content", **overrides: Any + ) -> tuple[ContextFragment, ContextFragment]: + """Create a pair of duplicate ContextFragments for deduplication testing. + + Args: + content: The shared content + **overrides: Override any other default values + + Returns: + A tuple of two ContextFragments with identical content + """ + frag1 = ContextFragmentFactory.create_with_content(content, **overrides) + frag2 = ContextFragmentFactory.create_with_content( + content, + uko_node=frag1.uko_node, + relevance_score=round(frag1.relevance_score - 0.1, 2), + **overrides, + ) + return (frag1, frag2) + + +class ContextBudgetFactory: + """Factory for creating realistic ContextBudget test objects.""" + + @staticmethod + def create( + max_tokens: int | None = None, + reserved_tokens: int | None = None, + ) -> ContextBudget: + """Create a ContextBudget with realistic default data. + + Args: + max_tokens: Maximum tokens (default: random 100-1000) + reserved_tokens: Reserved tokens (default: 0) + + Returns: + A ContextBudget instance + """ + if max_tokens is None: + max_tokens = _faker.random_int(min=100, max=1000) + if reserved_tokens is None: + reserved_tokens = 0 + return ContextBudget(max_tokens=max_tokens, reserved_tokens=reserved_tokens) + + @staticmethod + def create_tight() -> ContextBudget: + """Create a ContextBudget with tight constraints (100-200 tokens).""" + max_tokens = _faker.random_int(min=100, max=200) + return ContextBudgetFactory.create(max_tokens=max_tokens) + + @staticmethod + def create_generous() -> ContextBudget: + """Create a ContextBudget with generous constraints (500-2000 tokens).""" + max_tokens = _faker.random_int(min=500, max=2000) + return ContextBudgetFactory.create(max_tokens=max_tokens) diff --git a/features/test_data_loader.py b/features/test_data_loader.py new file mode 100644 index 000000000..2ef756e05 --- /dev/null +++ b/features/test_data_loader.py @@ -0,0 +1,92 @@ +"""Loader for externalized test data from JSON files. + +This module provides utilities to load and access test data that has been +externalized to JSON files for easier management and reuse across multiple +test scenarios. + +Usage: + from features.test_data_loader import TestDataLoader + + loader = TestDataLoader() + project_names = loader.get_project_names() + skill_names = loader.get_skill_names() + python_samples = loader.get_python_code_samples() +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class TestDataLoader: + """Load and provide access to externalized test data.""" + + _DATA_FILE = Path(__file__).parent / "fixtures" / "test_data_samples.json" + _cache: dict[str, Any] | None = None + + @classmethod + def _load_data(cls) -> dict[str, Any]: + """Load test data from JSON file (cached).""" + if cls._cache is None: + if not cls._DATA_FILE.exists(): + raise FileNotFoundError(f"Test data file not found: {cls._DATA_FILE}") + with open(cls._DATA_FILE) as f: + cls._cache = json.load(f) + return cls._cache + + @classmethod + def get_project_names(cls) -> list[str]: + """Get list of realistic project names.""" + return cls._load_data().get("project_names", []) + + @classmethod + def get_skill_names(cls) -> list[str]: + """Get list of realistic skill names.""" + return cls._load_data().get("skill_names", []) + + @classmethod + def get_tool_names(cls) -> list[str]: + """Get list of realistic tool names.""" + return cls._load_data().get("tool_names", []) + + @classmethod + def get_python_code_samples(cls) -> list[str]: + """Get list of realistic Python code samples.""" + return cls._load_data().get("python_code_samples", []) + + @classmethod + def get_file_paths(cls) -> list[str]: + """Get list of realistic file paths.""" + return cls._load_data().get("file_paths", []) + + @classmethod + def get_uko_node_uris(cls) -> list[str]: + """Get list of realistic UKO node URIs.""" + return cls._load_data().get("uko_node_uris", []) + + @classmethod + def get_resource_uris(cls) -> list[str]: + """Get list of realistic resource URIs.""" + return cls._load_data().get("resource_uris", []) + + @classmethod + def get_prose_samples(cls) -> list[str]: + """Get list of realistic prose samples.""" + return cls._load_data().get("prose_samples", []) + + @classmethod + def get_edge_case_names(cls) -> list[str]: + """Get list of edge case names for boundary testing.""" + return cls._load_data().get("edge_case_names", []) + + @classmethod + def get_invalid_special_chars(cls) -> list[str]: + """Get list of invalid special characters for validation testing.""" + return cls._load_data().get("invalid_special_chars", []) + + @classmethod + def get_all_data(cls) -> dict[str, Any]: + """Get all loaded test data.""" + return cls._load_data() diff --git a/pyproject.toml b/pyproject.toml index 7f8587b5f..d13fe9af3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ tests = [ "asv>=0.6.5", "robotframework>=7.3.2", "robotframework-pabot>=4.0.0", + "faker>=20.0.0", # Dynamic test data generation ] docs = [ "mkdocs>=1.6.1", diff --git a/robot/helper_acms_fusion.py b/robot/helper_acms_fusion.py index 36297b28b..c855db931 100644 --- a/robot/helper_acms_fusion.py +++ b/robot/helper_acms_fusion.py @@ -3,6 +3,9 @@ Provides a CLI-style interface for Robot to invoke StrategyCoordinator and FusionEngine operations and verify the results. +Uses dynamic test data generation via helper_test_data_factory for realistic +test data instead of hardcoded values. + Usage: python robot/helper_acms_fusion.py coord-basic python robot/helper_acms_fusion.py coord-budget @@ -26,7 +29,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src") if _SRC not in sys.path: sys.path.insert(0, _SRC) -from cleveragents.application.services.acms_pipeline import ( # noqa: E402 +from cleveragents.application.services.acms_pipeline import ( # noqa: E402, I001 CircuitBreaker, ParallelStrategyExecutor, ) @@ -44,17 +47,13 @@ from cleveragents.application.services.strategy_coordinator import ( # noqa: E4 from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 ContextBudget, ContextFragment, - FragmentProvenance, ) -_DEFAULT_PROV = FragmentProvenance(resource_uri="test://robot-fusion") - - -def _make_frag(**kwargs: Any) -> ContextFragment: - kwargs.setdefault("uko_node", "test://robot-fusion") - kwargs.setdefault("token_count", 10) - kwargs.setdefault("provenance", _DEFAULT_PROV) - return ContextFragment(**kwargs) +from helper_test_data_factory import ( # noqa: E402 + RobotContextBudgetFactory, + RobotContextFragmentFactory, + RobotTestDataGenerator, +) class _TestStrategy: @@ -93,18 +92,21 @@ class _TestStrategy: def _cmd_coord_basic() -> int: - """StrategyCoordinator basic coordination.""" + """StrategyCoordinator basic coordination with realistic test data.""" coordinator = StrategyCoordinator() strategies = [_TestStrategy("a", 0.8), _TestStrategy("b", 0.6)] + # Use factory to generate realistic fragments instead of hardcoded "alpha"/"beta" frags = [ - _make_frag( - uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.9, ), - _make_frag( - uko_node="p://b.py", content="beta", token_count=50, relevance_score=0.5 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.5, ), ] - b = ContextBudget(max_tokens=200, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=200) result = coordinator.coordinate( request={}, strategies=strategies, @@ -122,11 +124,12 @@ def _cmd_coord_budget() -> int: coordinator = StrategyCoordinator() strategies = [_TestStrategy("a", 0.8), _TestStrategy("b", 0.2)] frags = [ - _make_frag( - uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.9, ), ] - b = ContextBudget(max_tokens=1000, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=1000) result = coordinator.coordinate( request={}, strategies=strategies, @@ -147,11 +150,12 @@ def _cmd_coord_caps() -> int: coordinator = StrategyCoordinator(config=config) strategies = [_TestStrategy("a", 0.9), _TestStrategy("b", 0.1)] frags = [ - _make_frag( - uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.9, ), ] - b = ContextBudget(max_tokens=1000, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=1000) result = coordinator.coordinate( request={}, strategies=strategies, @@ -172,11 +176,12 @@ def _cmd_coord_circuit() -> int: coordinator = StrategyCoordinator(executor=executor) strategies = [_TestStrategy("broken", 0.7)] frags = [ - _make_frag( - uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.9, ), ] - b = ContextBudget(max_tokens=200, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=200) result = coordinator.coordinate( request={}, strategies=strategies, @@ -189,23 +194,30 @@ def _cmd_coord_circuit() -> int: def _cmd_fuse_dedup() -> int: - """FusionEngine deduplication.""" + """FusionEngine deduplication with realistic test data.""" engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1)) + # Create fragments with same content for deduplication testing + shared_content = RobotTestDataGenerator.python_code() + shared_uko = RobotTestDataGenerator.uko_node_uri() frags = [ - _make_frag( - uko_node="p://dup.py", content="same", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + uko_node=shared_uko, + content=shared_content, + token_count=50, + relevance_score=0.9, ), - _make_frag( - uko_node="p://dup.py", content="same", token_count=50, relevance_score=0.7 + RobotContextFragmentFactory.create_object( + uko_node=shared_uko, + content=shared_content, + token_count=50, + relevance_score=0.7, ), - _make_frag( - uko_node="p://other.py", - content="other", + RobotContextFragmentFactory.create_object( token_count=50, relevance_score=0.5, ), ] - b = ContextBudget(max_tokens=500, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=500) result = engine.fuse(frags, b) assert result.dedup_count > 0, f"dedup_count={result.dedup_count}" assert len(result.fragments) < len(frags), "Expected fewer fragments" @@ -214,37 +226,34 @@ def _cmd_fuse_dedup() -> int: def _cmd_fuse_depth() -> int: - """FusionEngine depth resolution.""" + """FusionEngine depth resolution with realistic test data.""" engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1)) + shared_uko = RobotTestDataGenerator.uko_node_uri() frags = [ - _make_frag( - uko_node="p://deep.py", - content="shallow", + RobotContextFragmentFactory.create_object( + uko_node=shared_uko, token_count=50, detail_depth=2, relevance_score=0.8, ), - _make_frag( - uko_node="p://deep.py", - content="deep detail", + RobotContextFragmentFactory.create_object( + uko_node=shared_uko, token_count=80, detail_depth=5, relevance_score=0.7, ), - _make_frag( - uko_node="p://other.py", - content="other", + RobotContextFragmentFactory.create_object( token_count=50, detail_depth=3, relevance_score=0.6, ), ] - b = ContextBudget(max_tokens=500, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=500) result = engine.fuse(frags, b) assert result.depth_resolved_count > 0, ( f"depth_resolved={result.depth_resolved_count}" ) - deep_frags = [f for f in result.fragments if "deep" in f.uko_node] + deep_frags = [f for f in result.fragments if f.detail_depth >= 5] if deep_frags: assert max(f.detail_depth for f in deep_frags) == 5 print("fuse-depth-ok") @@ -252,18 +261,16 @@ def _cmd_fuse_depth() -> int: def _cmd_fuse_pack() -> int: - """FusionEngine knapsack packing.""" + """FusionEngine knapsack packing with realistic test data.""" engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1)) frags = [ - _make_frag( - uko_node=f"p://f{i}.py", - content=f"c{i}", + RobotContextFragmentFactory.create_object( token_count=100, relevance_score=round(0.9 - i * 0.1, 2), ) for i in range(6) ] - b = ContextBudget(max_tokens=400, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=400) result = engine.fuse(frags, b) assert result.total_tokens <= 400, f"total_tokens={result.total_tokens}" print("fuse-pack-ok") @@ -271,19 +278,17 @@ def _cmd_fuse_pack() -> int: def _cmd_fuse_overage() -> int: - """FusionEngine budget overage guard.""" + """FusionEngine budget overage guard with realistic test data.""" config = FusionConfig(overage_guard_enabled=True, min_fragment_tokens=1) engine = FusionEngine(config=config) frags = [ - _make_frag( - uko_node=f"p://ov{i}.py", - content=f"ov{i}", + RobotContextFragmentFactory.create_object( token_count=40, relevance_score=round(0.9 - i * 0.2, 2), ) for i in range(4) ] - b = ContextBudget(max_tokens=100, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=100) result = engine.fuse(frags, b) assert result.total_tokens <= 100, f"total_tokens={result.total_tokens}" print("fuse-overage-ok") @@ -291,18 +296,20 @@ def _cmd_fuse_overage() -> int: def _cmd_integration() -> int: - """Integration: coordinator -> fusion.""" + """Integration: coordinator -> fusion with realistic test data.""" coordinator = StrategyCoordinator() strategies = [_TestStrategy("a", 0.8)] frags = [ - _make_frag( - uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.9, ), - _make_frag( - uko_node="p://b.py", content="beta", token_count=50, relevance_score=0.5 + RobotContextFragmentFactory.create_object( + token_count=50, + relevance_score=0.5, ), ] - b = ContextBudget(max_tokens=200, reserved_tokens=0) + b = RobotContextBudgetFactory.create_object(max_tokens=200) coord_result = coordinator.coordinate( request={}, strategies=strategies, diff --git a/robot/helper_test_data_factory.py b/robot/helper_test_data_factory.py new file mode 100644 index 000000000..0cb807327 --- /dev/null +++ b/robot/helper_test_data_factory.py @@ -0,0 +1,374 @@ +"""Robot Framework test data factory helper using Faker for realistic data generation. + +This module provides a CLI-style interface for Robot Framework to generate realistic +test data using Faker. It replaces hardcoded test data with dynamically generated, +varied, and realistic values. + +Usage: + python robot/helper_test_data_factory.py generate-fragment + python robot/helper_test_data_factory.py generate-fragments 5 + python robot/helper_test_data_factory.py generate-project-name + python robot/helper_test_data_factory.py generate-code + python robot/helper_test_data_factory.py generate-budget +""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from faker import Faker + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 + ContextBudget, + ContextFragment, + FragmentProvenance, +) + +_faker = Faker() + + +class RobotTestDataGenerator: + """Generate realistic test data for Robot Framework tests.""" + + @staticmethod + def project_name() -> str: + """Generate a realistic project name.""" + return _faker.word() + "-" + _faker.word() + + @staticmethod + def skill_name() -> str: + """Generate a realistic skill name.""" + return _faker.word() + "_" + _faker.word() + + @staticmethod + def tool_name() -> str: + """Generate a realistic tool name.""" + return _faker.word() + "-tool" + + @staticmethod + def actor_name() -> str: + """Generate a realistic actor name.""" + return _faker.first_name() + _faker.last_name() + + @staticmethod + def python_code() -> str: + """Generate a realistic Python code snippet.""" + var_name = _faker.word() + value = _faker.random_int(min=1, max=1000) + return f"{var_name} = {value}" + + @staticmethod + def python_prose() -> str: + """Generate realistic Python-like prose.""" + return _faker.sentence(nb_words=10) + + @staticmethod + def file_path() -> str: + """Generate a realistic file path.""" + return f"src/{_faker.word()}/{_faker.word()}.py" + + @staticmethod + def uko_node_uri() -> str: + """Generate a realistic UKO node URI.""" + return f"p://{_faker.word()}/{_faker.word()}.py" + + @staticmethod + def resource_uri() -> str: + """Generate a realistic resource URI.""" + return f"test://{_faker.word()}-{_faker.word()}" + + @staticmethod + def relevance_score() -> float: + """Generate a realistic relevance score (0.0 to 1.0).""" + return round(_faker.pyfloat(min_value=0.0, max_value=1.0, right_digits=2), 2) + + @staticmethod + def token_count() -> int: + """Generate a realistic token count.""" + return _faker.random_int(min=10, max=500) + + @staticmethod + def detail_depth() -> int: + """Generate a realistic detail depth level.""" + return _faker.random_int(min=1, max=10) + + @staticmethod + def ulid() -> str: + """Generate a valid ULID-like string (26 chars, base32).""" + valid_chars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + return "".join(_faker.random.choice(valid_chars) for _ in range(26)) + + +class RobotContextFragmentFactory: + """Factory for creating realistic ContextFragment test objects for Robot.""" + + @staticmethod + def create_dict(**overrides: Any) -> dict[str, Any]: + """Create a ContextFragment as a dictionary (for Robot serialization). + + Args: + **overrides: Override any default values + + Returns: + A dictionary representation of a ContextFragment + """ + defaults = { + "uko_node": RobotTestDataGenerator.uko_node_uri(), + "content": RobotTestDataGenerator.python_code(), + "token_count": RobotTestDataGenerator.token_count(), + "relevance_score": RobotTestDataGenerator.relevance_score(), + "detail_depth": RobotTestDataGenerator.detail_depth(), + "provenance": { + "resource_uri": RobotTestDataGenerator.resource_uri(), + }, + } + defaults.update(overrides) + return defaults + + @staticmethod + def create_object(**overrides: Any) -> ContextFragment: + """Create a ContextFragment object. + + Args: + **overrides: Override any default values + + Returns: + A ContextFragment instance + """ + data = RobotContextFragmentFactory.create_dict(**overrides) + provenance_data = data.pop("provenance") + provenance = FragmentProvenance(**provenance_data) + return ContextFragment(provenance=provenance, **data) + + @staticmethod + def create_batch_dicts(count: int, **overrides: Any) -> list[dict[str, Any]]: + """Create multiple ContextFragments as dictionaries. + + Args: + count: Number of fragments to create + **overrides: Override any default values + + Returns: + A list of dictionary representations + """ + return [ + RobotContextFragmentFactory.create_dict(**overrides) for _ in range(count) + ] + + @staticmethod + def create_batch_objects(count: int, **overrides: Any) -> list[ContextFragment]: + """Create multiple ContextFragment objects. + + Args: + count: Number of fragments to create + **overrides: Override any default values + + Returns: + A list of ContextFragment instances + """ + return [ + RobotContextFragmentFactory.create_object(**overrides) for _ in range(count) + ] + + +class RobotContextBudgetFactory: + """Factory for creating realistic ContextBudget test objects for Robot.""" + + @staticmethod + def create_dict( + max_tokens: int | None = None, + reserved_tokens: int | None = None, + ) -> dict[str, Any]: + """Create a ContextBudget as a dictionary. + + Args: + max_tokens: Maximum tokens (default: random 100-1000) + reserved_tokens: Reserved tokens (default: 0) + + Returns: + A dictionary representation of a ContextBudget + """ + if max_tokens is None: + max_tokens = _faker.random_int(min=100, max=1000) + if reserved_tokens is None: + reserved_tokens = 0 + return { + "max_tokens": max_tokens, + "reserved_tokens": reserved_tokens, + } + + @staticmethod + def create_object( + max_tokens: int | None = None, + reserved_tokens: int | None = None, + ) -> ContextBudget: + """Create a ContextBudget object. + + Args: + max_tokens: Maximum tokens (default: random 100-1000) + reserved_tokens: Reserved tokens (default: 0) + + Returns: + A ContextBudget instance + """ + data = RobotContextBudgetFactory.create_dict(max_tokens, reserved_tokens) + return ContextBudget(**data) + + +# --------------------------------------------------------------------------- +# CLI commands for Robot Framework +# --------------------------------------------------------------------------- + + +def _cmd_generate_fragment() -> int: + """Generate a single context fragment as JSON.""" + frag_dict = RobotContextFragmentFactory.create_dict() + print(json.dumps(frag_dict, indent=2)) + return 0 + + +def _cmd_generate_fragments() -> int: + """Generate multiple context fragments as JSON.""" + count = int(sys.argv[2]) if len(sys.argv) > 2 else 5 + frags = RobotContextFragmentFactory.create_batch_dicts(count) + print(json.dumps(frags, indent=2)) + return 0 + + +def _cmd_generate_project_name() -> int: + """Generate a realistic project name.""" + print(RobotTestDataGenerator.project_name()) + return 0 + + +def _cmd_generate_skill_name() -> int: + """Generate a realistic skill name.""" + print(RobotTestDataGenerator.skill_name()) + return 0 + + +def _cmd_generate_tool_name() -> int: + """Generate a realistic tool name.""" + print(RobotTestDataGenerator.tool_name()) + return 0 + + +def _cmd_generate_actor_name() -> int: + """Generate a realistic actor name.""" + print(RobotTestDataGenerator.actor_name()) + return 0 + + +def _cmd_generate_code() -> int: + """Generate a realistic Python code snippet.""" + print(RobotTestDataGenerator.python_code()) + return 0 + + +def _cmd_generate_prose() -> int: + """Generate realistic Python-like prose.""" + print(RobotTestDataGenerator.python_prose()) + return 0 + + +def _cmd_generate_file_path() -> int: + """Generate a realistic file path.""" + print(RobotTestDataGenerator.file_path()) + return 0 + + +def _cmd_generate_uko_uri() -> int: + """Generate a realistic UKO node URI.""" + print(RobotTestDataGenerator.uko_node_uri()) + return 0 + + +def _cmd_generate_resource_uri() -> int: + """Generate a realistic resource URI.""" + print(RobotTestDataGenerator.resource_uri()) + return 0 + + +def _cmd_generate_relevance_score() -> int: + """Generate a realistic relevance score.""" + print(RobotTestDataGenerator.relevance_score()) + return 0 + + +def _cmd_generate_token_count() -> int: + """Generate a realistic token count.""" + print(RobotTestDataGenerator.token_count()) + return 0 + + +def _cmd_generate_detail_depth() -> int: + """Generate a realistic detail depth.""" + print(RobotTestDataGenerator.detail_depth()) + return 0 + + +def _cmd_generate_ulid() -> int: + """Generate a valid ULID.""" + print(RobotTestDataGenerator.ulid()) + return 0 + + +def _cmd_generate_budget() -> int: + """Generate a context budget as JSON.""" + budget_dict = RobotContextBudgetFactory.create_dict() + print(json.dumps(budget_dict, indent=2)) + return 0 + + +# --------------------------------------------------------------------------- +# CLI dispatcher +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], int]] = { + "generate-fragment": _cmd_generate_fragment, + "generate-fragments": _cmd_generate_fragments, + "generate-project-name": _cmd_generate_project_name, + "generate-skill-name": _cmd_generate_skill_name, + "generate-tool-name": _cmd_generate_tool_name, + "generate-actor-name": _cmd_generate_actor_name, + "generate-code": _cmd_generate_code, + "generate-prose": _cmd_generate_prose, + "generate-file-path": _cmd_generate_file_path, + "generate-uko-uri": _cmd_generate_uko_uri, + "generate-resource-uri": _cmd_generate_resource_uri, + "generate-relevance-score": _cmd_generate_relevance_score, + "generate-token-count": _cmd_generate_token_count, + "generate-detail-depth": _cmd_generate_detail_depth, + "generate-ulid": _cmd_generate_ulid, + "generate-budget": _cmd_generate_budget, +} + + +def main() -> int: + """Main entry point for CLI.""" + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print( + f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>", + file=sys.stderr, + ) + return 1 + try: + return _COMMANDS[sys.argv[1]]() + except Exception as exc: + print(f"FAIL: {exc}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) -- 2.52.0