test(data): introduce dynamic data generation and externalize test data in Behave and Robot Framework suites #9254

Merged
HAL9000 merged 2 commits from test/improve-test-data-quality-realism into master 2026-04-27 07:46:53 +00:00
9 changed files with 963 additions and 82 deletions
+9 -12
View File
@@ -35,12 +35,11 @@ Feature: Consolidated Routing
# Feature: Reactive registry adapter coverage
# ============================================================
Scenario: Registry list failure is ignored
Scenario: Registry list failure propagates to caller
Given a reactive stream router and route bridge
And an actor registry that raises on list
When I register registry actors with the adapter
Then no agents are registered
And the route bridge agents remain unchanged
When I attempt to register registry actors with the adapter
Then a RuntimeError is raised
Scenario: Registers only actors with processing methods
@@ -51,20 +50,18 @@ Feature: Consolidated Routing
And the route bridge agents cache includes all listed actors
Scenario: Route bridge cache resets when refresh fails
Scenario: Route bridge refresh failure propagates to caller
Given a reactive stream router and route bridge
And an actor registry that returns actors without names
When I register registry actors with the adapter
Then the route bridge agents cache is cleared
And no agents are registered
When I attempt to register registry actors with the adapter
Then an AttributeError is raised
Scenario: Route bridge cache resets when registry returns none
Scenario: Route bridge refresh failure propagates when registry returns none
Given a reactive stream router and route bridge
And an actor registry that returns none
When I register registry actors with the adapter
Then the route bridge agents cache is cleared
And no agents are registered
When I attempt to register registry actors with the adapter
Then a TypeError is raised
# ============================================================
+132
View File
@@ -0,0 +1,132 @@
{
"project_names": [
"data-pipeline",
"ml-framework",
"api-gateway",
"auth-service",
"cache-layer",
"event-stream",
"config-manager",
"logging-system",
"monitoring-tool",
"deployment-engine"
],
"skill_names": [
"text_analysis",
"code_generation",
"data_validation",
"error_handling",
"performance_tuning",
"security_audit",
"documentation_gen",
"test_generation",
"refactoring_assist",
"debugging_help"
],
"tool_names": [
"file-reader",
"api-caller",
"database-query",
"cache-store",
"message-queue",
"webhook-sender",
"log-aggregator",
"metric-collector",
"config-loader",
"secret-manager"
],
"python_code_samples": [
"def hello_world():\n return 'Hello, World!'",
"result = sum([1, 2, 3, 4, 5])",
"data = {'name': 'Alice', 'age': 30}",
"for i in range(10):\n print(i)",
"import json\ndata = json.loads('{}')",
"class Calculator:\n def add(self, a, b):\n return a + b",
"async def fetch_data():\n return await get_response()",
"try:\n result = risky_operation()\nexcept Exception as e:\n print(e)",
"list_comp = [x * 2 for x in range(5)]",
"lambda_func = lambda x: x ** 2"
],
"file_paths": [
"src/core/models.py",
"src/services/auth.py",
"src/handlers/api.py",
"src/utils/helpers.py",
"src/config/settings.py",
"tests/unit/test_models.py",
"tests/integration/test_api.py",
"docs/api_reference.md",
"scripts/migrate_db.py",
"examples/basic_usage.py"
],
"uko_node_uris": [
"p://src/models.py",
"p://src/services/auth.py",
"p://src/handlers/api.py",
"p://src/utils/helpers.py",
"p://src/config/settings.py",
"p://tests/unit/test_models.py",
"p://tests/integration/test_api.py",
"p://docs/api_reference.md",
"p://scripts/migrate_db.py",
"p://examples/basic_usage.py"
],
"resource_uris": [
"test://database-primary",
"test://cache-redis",
"test://message-queue",
"test://api-gateway",
"test://auth-service",
"test://logging-system",
"test://monitoring-tool",
"test://config-store",
"test://secret-vault",
"test://deployment-engine"
],
"prose_samples": [
"This function implements a robust error handling mechanism.",
"The API endpoint validates all incoming requests before processing.",
"Database queries are optimized for performance and scalability.",
"The authentication system uses industry-standard security practices.",
"Configuration management supports multiple environments seamlessly.",
"Logging is comprehensive and includes detailed diagnostic information.",
"Monitoring tools provide real-time visibility into system health.",
"The caching layer reduces database load significantly.",
"Message queues ensure reliable asynchronous communication.",
"Deployment automation minimizes manual intervention and errors."
],
"edge_case_names": [
"a",
"my-project",
"my_project",
"my project",
"project123",
"123project",
"My Project Name",
"UPPERCASE",
"lowercase",
"MixedCase"
],
"invalid_special_chars": [
"@",
"/",
"!",
"#",
"%",
"^",
"&",
"*",
"+",
"=",
"<",
">",
"?",
".",
":",
";",
"|",
"\\",
"`",
"~"
]
}
@@ -61,6 +61,18 @@ def step_register_actors(context):
)
@when("I attempt to register registry actors with the adapter")
def step_attempt_register_actors(context):
"""Attempt registration, capturing any exception that propagates."""
context.raised_exception = None
try:
register_registry_agents(
context.stream_router, context.route_bridge, context.actor_registry
)
except Exception as exc:
context.raised_exception = exc
@then("no agents are registered")
def step_assert_no_agents(context):
assert context.stream_router.agents == {}
@@ -88,3 +100,36 @@ def step_assert_bridge_cache(context):
@then("the route bridge agents cache is cleared")
def step_assert_bridge_cleared(context):
assert context.route_bridge.agents == {}
@then("a RuntimeError is raised")
def step_assert_runtime_error(context):
assert context.raised_exception is not None, (
"Expected RuntimeError but no exception was raised"
)
assert isinstance(context.raised_exception, RuntimeError), (
f"Expected RuntimeError but got {type(context.raised_exception).__name__}: "
f"{context.raised_exception}"
)
@then("an AttributeError is raised")
def step_assert_attribute_error(context):
assert context.raised_exception is not None, (
"Expected AttributeError but no exception was raised"
)
assert isinstance(context.raised_exception, AttributeError), (
f"Expected AttributeError but got {type(context.raised_exception).__name__}: "
f"{context.raised_exception}"
)
@then("a TypeError is raised")
def step_assert_type_error(context):
assert context.raised_exception is not None, (
"Expected TypeError but no exception was raised"
)
assert isinstance(context.raised_exception, TypeError), (
f"Expected TypeError but got {type(context.raised_exception).__name__}: "
f"{context.raised_exception}"
)
+236
View File
@@ -0,0 +1,236 @@
"""Test data factory for Behave tests using Faker for realistic data generation.
This module provides factory functions and classes for generating realistic test data
for Behave scenarios. It uses Faker to generate dynamic, varied test data instead of
hardcoded values, improving test realism and maintainability.
Usage:
from features.test_data_factory import ContextFragmentFactory, TestDataGenerator
# Generate a single context fragment with realistic data
fragment = ContextFragmentFactory.create()
# Generate multiple fragments with varied data
fragments = ContextFragmentFactory.create_batch(5)
# Generate realistic project names
project_name = TestDataGenerator.project_name()
# Generate realistic Python code snippets
code = TestDataGenerator.python_code()
"""
from __future__ import annotations
import sys
from pathlib import Path
from typing import Any
from faker import Faker
# Add src to path for imports
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
ContextBudget,
ContextFragment,
FragmentProvenance,
)
# Initialize Faker for consistent, reproducible test data
_faker = Faker()
class TestDataGenerator:
"""Generate realistic test data for various domain objects."""
@staticmethod
def project_name() -> str:
"""Generate a realistic project name."""
return _faker.word() + "-" + _faker.word()
@staticmethod
def skill_name() -> str:
"""Generate a realistic skill name."""
return _faker.word() + "_" + _faker.word()
@staticmethod
def tool_name() -> str:
"""Generate a realistic tool name."""
return _faker.word() + "-tool"
@staticmethod
def actor_name() -> str:
"""Generate a realistic actor name."""
return _faker.first_name() + _faker.last_name()
@staticmethod
def python_code() -> str:
"""Generate a realistic Python code snippet."""
var_name = _faker.word()
value = _faker.random_int(min=1, max=1000)
return f"{var_name} = {value}"
@staticmethod
def python_prose() -> str:
"""Generate realistic Python-like prose."""
return _faker.sentence(nb_words=10)
@staticmethod
def file_path() -> str:
"""Generate a realistic file path."""
return f"src/{_faker.word()}/{_faker.word()}.py"
@staticmethod
def uko_node_uri() -> str:
"""Generate a realistic UKO node URI."""
return f"p://{_faker.word()}/{_faker.word()}.py"
@staticmethod
def resource_uri() -> str:
"""Generate a realistic resource URI."""
return f"test://{_faker.word()}-{_faker.word()}"
@staticmethod
def namespaced_name() -> tuple[str, str]:
"""Generate a realistic namespaced name (namespace, name)."""
namespace = _faker.word()
name = _faker.word() + "-" + _faker.word()
return (namespace, name)
@staticmethod
def ulid() -> str:
"""Generate a valid ULID-like string (26 chars, base32)."""
# Generate a valid Crockford base32 ULID (26 characters)
# Using only valid characters: 0-9, A-Z (excluding I, L, O, U)
valid_chars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
return "".join(_faker.random.choice(valid_chars) for _ in range(26))
@staticmethod
def relevance_score() -> float:
"""Generate a realistic relevance score (0.0 to 1.0)."""
return round(_faker.pyfloat(min_value=0.0, max_value=1.0, right_digits=2), 2)
@staticmethod
def token_count() -> int:
"""Generate a realistic token count."""
return _faker.random_int(min=10, max=500)
@staticmethod
def detail_depth() -> int:
"""Generate a realistic detail depth level."""
return _faker.random_int(min=1, max=10)
class ContextFragmentFactory:
"""Factory for creating realistic ContextFragment test objects."""
@staticmethod
def create(**overrides: Any) -> ContextFragment:
"""Create a single ContextFragment with realistic default data.
Args:
**overrides: Override any default values
Returns:
A ContextFragment instance with realistic data
"""
defaults = {
"uko_node": TestDataGenerator.uko_node_uri(),
"content": TestDataGenerator.python_code(),
"token_count": TestDataGenerator.token_count(),
"relevance_score": TestDataGenerator.relevance_score(),
"detail_depth": TestDataGenerator.detail_depth(),
"provenance": FragmentProvenance(
resource_uri=TestDataGenerator.resource_uri()
),
}
defaults.update(overrides)
return ContextFragment(**defaults)
@staticmethod
def create_batch(count: int, **overrides: Any) -> list[ContextFragment]:
"""Create multiple ContextFragments with varied realistic data.
Args:
count: Number of fragments to create
**overrides: Override any default values (applied to all)
Returns:
A list of ContextFragment instances
"""
return [ContextFragmentFactory.create(**overrides) for _ in range(count)]
@staticmethod
def create_with_content(content: str, **overrides: Any) -> ContextFragment:
"""Create a ContextFragment with specific content.
Args:
content: The content to use
**overrides: Override any other default values
Returns:
A ContextFragment instance with the specified content
"""
overrides["content"] = content
return ContextFragmentFactory.create(**overrides)
@staticmethod
def create_duplicate_pair(
content: str = "duplicate_content", **overrides: Any
) -> tuple[ContextFragment, ContextFragment]:
"""Create a pair of duplicate ContextFragments for deduplication testing.
Args:
content: The shared content
**overrides: Override any other default values
Returns:
A tuple of two ContextFragments with identical content
"""
frag1 = ContextFragmentFactory.create_with_content(content, **overrides)
frag2 = ContextFragmentFactory.create_with_content(
content,
uko_node=frag1.uko_node,
relevance_score=round(frag1.relevance_score - 0.1, 2),
**overrides,
)
return (frag1, frag2)
class ContextBudgetFactory:
"""Factory for creating realistic ContextBudget test objects."""
@staticmethod
def create(
max_tokens: int | None = None,
reserved_tokens: int | None = None,
) -> ContextBudget:
"""Create a ContextBudget with realistic default data.
Args:
max_tokens: Maximum tokens (default: random 100-1000)
reserved_tokens: Reserved tokens (default: 0)
Returns:
A ContextBudget instance
"""
if max_tokens is None:
max_tokens = _faker.random_int(min=100, max=1000)
if reserved_tokens is None:
reserved_tokens = 0
return ContextBudget(max_tokens=max_tokens, reserved_tokens=reserved_tokens)
@staticmethod
def create_tight() -> ContextBudget:
"""Create a ContextBudget with tight constraints (100-200 tokens)."""
max_tokens = _faker.random_int(min=100, max=200)
return ContextBudgetFactory.create(max_tokens=max_tokens)
@staticmethod
def create_generous() -> ContextBudget:
"""Create a ContextBudget with generous constraints (500-2000 tokens)."""
max_tokens = _faker.random_int(min=500, max=2000)
return ContextBudgetFactory.create(max_tokens=max_tokens)
+92
View File
@@ -0,0 +1,92 @@
"""Loader for externalized test data from JSON files.
This module provides utilities to load and access test data that has been
externalized to JSON files for easier management and reuse across multiple
test scenarios.
Usage:
from features.test_data_loader import TestDataLoader
loader = TestDataLoader()
project_names = loader.get_project_names()
skill_names = loader.get_skill_names()
python_samples = loader.get_python_code_samples()
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
class TestDataLoader:
"""Load and provide access to externalized test data."""
_DATA_FILE = Path(__file__).parent / "fixtures" / "test_data_samples.json"
_cache: dict[str, Any] | None = None
@classmethod
def _load_data(cls) -> dict[str, Any]:
"""Load test data from JSON file (cached)."""
if cls._cache is None:
if not cls._DATA_FILE.exists():
raise FileNotFoundError(f"Test data file not found: {cls._DATA_FILE}")
with open(cls._DATA_FILE) as f:
cls._cache = json.load(f)
return cls._cache
@classmethod
def get_project_names(cls) -> list[str]:
"""Get list of realistic project names."""
return cls._load_data().get("project_names", [])
@classmethod
def get_skill_names(cls) -> list[str]:
"""Get list of realistic skill names."""
return cls._load_data().get("skill_names", [])
@classmethod
def get_tool_names(cls) -> list[str]:
"""Get list of realistic tool names."""
return cls._load_data().get("tool_names", [])
@classmethod
def get_python_code_samples(cls) -> list[str]:
"""Get list of realistic Python code samples."""
return cls._load_data().get("python_code_samples", [])
@classmethod
def get_file_paths(cls) -> list[str]:
"""Get list of realistic file paths."""
return cls._load_data().get("file_paths", [])
@classmethod
def get_uko_node_uris(cls) -> list[str]:
"""Get list of realistic UKO node URIs."""
return cls._load_data().get("uko_node_uris", [])
@classmethod
def get_resource_uris(cls) -> list[str]:
"""Get list of realistic resource URIs."""
return cls._load_data().get("resource_uris", [])
@classmethod
def get_prose_samples(cls) -> list[str]:
"""Get list of realistic prose samples."""
return cls._load_data().get("prose_samples", [])
@classmethod
def get_edge_case_names(cls) -> list[str]:
"""Get list of edge case names for boundary testing."""
return cls._load_data().get("edge_case_names", [])
@classmethod
def get_invalid_special_chars(cls) -> list[str]:
"""Get list of invalid special characters for validation testing."""
return cls._load_data().get("invalid_special_chars", [])
@classmethod
def get_all_data(cls) -> dict[str, Any]:
"""Get all loaded test data."""
return cls._load_data()
+1
View File
@@ -83,6 +83,7 @@ tests = [
"asv>=0.6.5",
"robotframework>=7.3.2",
"robotframework-pabot>=4.0.0",
"faker>=20.0.0", # Dynamic test data generation
]
docs = [
"mkdocs>=1.6.1",
+69 -62
View File
@@ -3,6 +3,9 @@
Provides a CLI-style interface for Robot to invoke StrategyCoordinator
and FusionEngine operations and verify the results.
Uses dynamic test data generation via helper_test_data_factory for realistic
test data instead of hardcoded values.
Usage:
python robot/helper_acms_fusion.py coord-basic
python robot/helper_acms_fusion.py coord-budget
@@ -26,7 +29,7 @@ _SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.application.services.acms_pipeline import ( # noqa: E402
from cleveragents.application.services.acms_pipeline import ( # noqa: E402, I001
CircuitBreaker,
ParallelStrategyExecutor,
)
@@ -44,17 +47,13 @@ from cleveragents.application.services.strategy_coordinator import ( # noqa: E4
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
ContextBudget,
ContextFragment,
FragmentProvenance,
)
_DEFAULT_PROV = FragmentProvenance(resource_uri="test://robot-fusion")
def _make_frag(**kwargs: Any) -> ContextFragment:
kwargs.setdefault("uko_node", "test://robot-fusion")
kwargs.setdefault("token_count", 10)
kwargs.setdefault("provenance", _DEFAULT_PROV)
return ContextFragment(**kwargs)
from helper_test_data_factory import ( # noqa: E402
RobotContextBudgetFactory,
RobotContextFragmentFactory,
RobotTestDataGenerator,
)
class _TestStrategy:
@@ -93,18 +92,21 @@ class _TestStrategy:
def _cmd_coord_basic() -> int:
"""StrategyCoordinator basic coordination."""
"""StrategyCoordinator basic coordination with realistic test data."""
coordinator = StrategyCoordinator()
strategies = [_TestStrategy("a", 0.8), _TestStrategy("b", 0.6)]
# Use factory to generate realistic fragments instead of hardcoded "alpha"/"beta"
frags = [
_make_frag(
uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.9,
),
_make_frag(
uko_node="p://b.py", content="beta", token_count=50, relevance_score=0.5
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.5,
),
]
b = ContextBudget(max_tokens=200, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=200)
result = coordinator.coordinate(
request={},
strategies=strategies,
@@ -122,11 +124,12 @@ def _cmd_coord_budget() -> int:
coordinator = StrategyCoordinator()
strategies = [_TestStrategy("a", 0.8), _TestStrategy("b", 0.2)]
frags = [
_make_frag(
uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.9,
),
]
b = ContextBudget(max_tokens=1000, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=1000)
result = coordinator.coordinate(
request={},
strategies=strategies,
@@ -147,11 +150,12 @@ def _cmd_coord_caps() -> int:
coordinator = StrategyCoordinator(config=config)
strategies = [_TestStrategy("a", 0.9), _TestStrategy("b", 0.1)]
frags = [
_make_frag(
uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.9,
),
]
b = ContextBudget(max_tokens=1000, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=1000)
result = coordinator.coordinate(
request={},
strategies=strategies,
@@ -172,11 +176,12 @@ def _cmd_coord_circuit() -> int:
coordinator = StrategyCoordinator(executor=executor)
strategies = [_TestStrategy("broken", 0.7)]
frags = [
_make_frag(
uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.9,
),
]
b = ContextBudget(max_tokens=200, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=200)
result = coordinator.coordinate(
request={},
strategies=strategies,
@@ -189,23 +194,30 @@ def _cmd_coord_circuit() -> int:
def _cmd_fuse_dedup() -> int:
"""FusionEngine deduplication."""
"""FusionEngine deduplication with realistic test data."""
engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1))
# Create fragments with same content for deduplication testing
shared_content = RobotTestDataGenerator.python_code()
shared_uko = RobotTestDataGenerator.uko_node_uri()
frags = [
_make_frag(
uko_node="p://dup.py", content="same", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
uko_node=shared_uko,
content=shared_content,
token_count=50,
relevance_score=0.9,
),
_make_frag(
uko_node="p://dup.py", content="same", token_count=50, relevance_score=0.7
RobotContextFragmentFactory.create_object(
uko_node=shared_uko,
content=shared_content,
token_count=50,
relevance_score=0.7,
),
_make_frag(
uko_node="p://other.py",
content="other",
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.5,
),
]
b = ContextBudget(max_tokens=500, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=500)
result = engine.fuse(frags, b)
assert result.dedup_count > 0, f"dedup_count={result.dedup_count}"
assert len(result.fragments) < len(frags), "Expected fewer fragments"
@@ -214,37 +226,34 @@ def _cmd_fuse_dedup() -> int:
def _cmd_fuse_depth() -> int:
"""FusionEngine depth resolution."""
"""FusionEngine depth resolution with realistic test data."""
engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1))
shared_uko = RobotTestDataGenerator.uko_node_uri()
frags = [
_make_frag(
uko_node="p://deep.py",
content="shallow",
RobotContextFragmentFactory.create_object(
uko_node=shared_uko,
token_count=50,
detail_depth=2,
relevance_score=0.8,
),
_make_frag(
uko_node="p://deep.py",
content="deep detail",
RobotContextFragmentFactory.create_object(
uko_node=shared_uko,
token_count=80,
detail_depth=5,
relevance_score=0.7,
),
_make_frag(
uko_node="p://other.py",
content="other",
RobotContextFragmentFactory.create_object(
token_count=50,
detail_depth=3,
relevance_score=0.6,
),
]
b = ContextBudget(max_tokens=500, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=500)
result = engine.fuse(frags, b)
assert result.depth_resolved_count > 0, (
f"depth_resolved={result.depth_resolved_count}"
)
deep_frags = [f for f in result.fragments if "deep" in f.uko_node]
deep_frags = [f for f in result.fragments if f.detail_depth >= 5]
if deep_frags:
assert max(f.detail_depth for f in deep_frags) == 5
print("fuse-depth-ok")
@@ -252,18 +261,16 @@ def _cmd_fuse_depth() -> int:
def _cmd_fuse_pack() -> int:
"""FusionEngine knapsack packing."""
"""FusionEngine knapsack packing with realistic test data."""
engine = FusionEngine(config=FusionConfig(min_fragment_tokens=1))
frags = [
_make_frag(
uko_node=f"p://f{i}.py",
content=f"c{i}",
RobotContextFragmentFactory.create_object(
token_count=100,
relevance_score=round(0.9 - i * 0.1, 2),
)
for i in range(6)
]
b = ContextBudget(max_tokens=400, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=400)
result = engine.fuse(frags, b)
assert result.total_tokens <= 400, f"total_tokens={result.total_tokens}"
print("fuse-pack-ok")
@@ -271,19 +278,17 @@ def _cmd_fuse_pack() -> int:
def _cmd_fuse_overage() -> int:
"""FusionEngine budget overage guard."""
"""FusionEngine budget overage guard with realistic test data."""
config = FusionConfig(overage_guard_enabled=True, min_fragment_tokens=1)
engine = FusionEngine(config=config)
frags = [
_make_frag(
uko_node=f"p://ov{i}.py",
content=f"ov{i}",
RobotContextFragmentFactory.create_object(
token_count=40,
relevance_score=round(0.9 - i * 0.2, 2),
)
for i in range(4)
]
b = ContextBudget(max_tokens=100, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=100)
result = engine.fuse(frags, b)
assert result.total_tokens <= 100, f"total_tokens={result.total_tokens}"
print("fuse-overage-ok")
@@ -291,18 +296,20 @@ def _cmd_fuse_overage() -> int:
def _cmd_integration() -> int:
"""Integration: coordinator -> fusion."""
"""Integration: coordinator -> fusion with realistic test data."""
coordinator = StrategyCoordinator()
strategies = [_TestStrategy("a", 0.8)]
frags = [
_make_frag(
uko_node="p://a.py", content="alpha", token_count=50, relevance_score=0.9
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.9,
),
_make_frag(
uko_node="p://b.py", content="beta", token_count=50, relevance_score=0.5
RobotContextFragmentFactory.create_object(
token_count=50,
relevance_score=0.5,
),
]
b = ContextBudget(max_tokens=200, reserved_tokens=0)
b = RobotContextBudgetFactory.create_object(max_tokens=200)
coord_result = coordinator.coordinate(
request={},
strategies=strategies,
+374
View File
@@ -0,0 +1,374 @@
"""Robot Framework test data factory helper using Faker for realistic data generation.
This module provides a CLI-style interface for Robot Framework to generate realistic
test data using Faker. It replaces hardcoded test data with dynamically generated,
varied, and realistic values.
Usage:
python robot/helper_test_data_factory.py generate-fragment
python robot/helper_test_data_factory.py generate-fragments 5
python robot/helper_test_data_factory.py generate-project-name
python robot/helper_test_data_factory.py generate-code
python robot/helper_test_data_factory.py generate-budget
"""
from __future__ import annotations
import json
import sys
from collections.abc import Callable
from pathlib import Path
from typing import Any
from faker import Faker
_SRC = str(Path(__file__).resolve().parents[1] / "src")
if _SRC not in sys.path:
sys.path.insert(0, _SRC)
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
ContextBudget,
ContextFragment,
FragmentProvenance,
)
_faker = Faker()
class RobotTestDataGenerator:
"""Generate realistic test data for Robot Framework tests."""
@staticmethod
def project_name() -> str:
"""Generate a realistic project name."""
return _faker.word() + "-" + _faker.word()
@staticmethod
def skill_name() -> str:
"""Generate a realistic skill name."""
return _faker.word() + "_" + _faker.word()
@staticmethod
def tool_name() -> str:
"""Generate a realistic tool name."""
return _faker.word() + "-tool"
@staticmethod
def actor_name() -> str:
"""Generate a realistic actor name."""
return _faker.first_name() + _faker.last_name()
@staticmethod
def python_code() -> str:
"""Generate a realistic Python code snippet."""
var_name = _faker.word()
value = _faker.random_int(min=1, max=1000)
return f"{var_name} = {value}"
@staticmethod
def python_prose() -> str:
"""Generate realistic Python-like prose."""
return _faker.sentence(nb_words=10)
@staticmethod
def file_path() -> str:
"""Generate a realistic file path."""
return f"src/{_faker.word()}/{_faker.word()}.py"
@staticmethod
def uko_node_uri() -> str:
"""Generate a realistic UKO node URI."""
return f"p://{_faker.word()}/{_faker.word()}.py"
@staticmethod
def resource_uri() -> str:
"""Generate a realistic resource URI."""
return f"test://{_faker.word()}-{_faker.word()}"
@staticmethod
def relevance_score() -> float:
"""Generate a realistic relevance score (0.0 to 1.0)."""
return round(_faker.pyfloat(min_value=0.0, max_value=1.0, right_digits=2), 2)
@staticmethod
def token_count() -> int:
"""Generate a realistic token count."""
return _faker.random_int(min=10, max=500)
@staticmethod
def detail_depth() -> int:
"""Generate a realistic detail depth level."""
return _faker.random_int(min=1, max=10)
@staticmethod
def ulid() -> str:
"""Generate a valid ULID-like string (26 chars, base32)."""
valid_chars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
return "".join(_faker.random.choice(valid_chars) for _ in range(26))
class RobotContextFragmentFactory:
"""Factory for creating realistic ContextFragment test objects for Robot."""
@staticmethod
def create_dict(**overrides: Any) -> dict[str, Any]:
"""Create a ContextFragment as a dictionary (for Robot serialization).
Args:
**overrides: Override any default values
Returns:
A dictionary representation of a ContextFragment
"""
defaults = {
"uko_node": RobotTestDataGenerator.uko_node_uri(),
"content": RobotTestDataGenerator.python_code(),
"token_count": RobotTestDataGenerator.token_count(),
"relevance_score": RobotTestDataGenerator.relevance_score(),
"detail_depth": RobotTestDataGenerator.detail_depth(),
"provenance": {
"resource_uri": RobotTestDataGenerator.resource_uri(),
},
}
defaults.update(overrides)
return defaults
@staticmethod
def create_object(**overrides: Any) -> ContextFragment:
"""Create a ContextFragment object.
Args:
**overrides: Override any default values
Returns:
A ContextFragment instance
"""
data = RobotContextFragmentFactory.create_dict(**overrides)
provenance_data = data.pop("provenance")
provenance = FragmentProvenance(**provenance_data)
return ContextFragment(provenance=provenance, **data)
@staticmethod
def create_batch_dicts(count: int, **overrides: Any) -> list[dict[str, Any]]:
"""Create multiple ContextFragments as dictionaries.
Args:
count: Number of fragments to create
**overrides: Override any default values
Returns:
A list of dictionary representations
"""
return [
RobotContextFragmentFactory.create_dict(**overrides) for _ in range(count)
]
@staticmethod
def create_batch_objects(count: int, **overrides: Any) -> list[ContextFragment]:
"""Create multiple ContextFragment objects.
Args:
count: Number of fragments to create
**overrides: Override any default values
Returns:
A list of ContextFragment instances
"""
return [
RobotContextFragmentFactory.create_object(**overrides) for _ in range(count)
]
class RobotContextBudgetFactory:
"""Factory for creating realistic ContextBudget test objects for Robot."""
@staticmethod
def create_dict(
max_tokens: int | None = None,
reserved_tokens: int | None = None,
) -> dict[str, Any]:
"""Create a ContextBudget as a dictionary.
Args:
max_tokens: Maximum tokens (default: random 100-1000)
reserved_tokens: Reserved tokens (default: 0)
Returns:
A dictionary representation of a ContextBudget
"""
if max_tokens is None:
max_tokens = _faker.random_int(min=100, max=1000)
if reserved_tokens is None:
reserved_tokens = 0
return {
"max_tokens": max_tokens,
"reserved_tokens": reserved_tokens,
}
@staticmethod
def create_object(
max_tokens: int | None = None,
reserved_tokens: int | None = None,
) -> ContextBudget:
"""Create a ContextBudget object.
Args:
max_tokens: Maximum tokens (default: random 100-1000)
reserved_tokens: Reserved tokens (default: 0)
Returns:
A ContextBudget instance
"""
data = RobotContextBudgetFactory.create_dict(max_tokens, reserved_tokens)
return ContextBudget(**data)
# ---------------------------------------------------------------------------
# CLI commands for Robot Framework
# ---------------------------------------------------------------------------
def _cmd_generate_fragment() -> int:
"""Generate a single context fragment as JSON."""
frag_dict = RobotContextFragmentFactory.create_dict()
print(json.dumps(frag_dict, indent=2))
return 0
def _cmd_generate_fragments() -> int:
"""Generate multiple context fragments as JSON."""
count = int(sys.argv[2]) if len(sys.argv) > 2 else 5
frags = RobotContextFragmentFactory.create_batch_dicts(count)
print(json.dumps(frags, indent=2))
return 0
def _cmd_generate_project_name() -> int:
"""Generate a realistic project name."""
print(RobotTestDataGenerator.project_name())
return 0
def _cmd_generate_skill_name() -> int:
"""Generate a realistic skill name."""
print(RobotTestDataGenerator.skill_name())
return 0
def _cmd_generate_tool_name() -> int:
"""Generate a realistic tool name."""
print(RobotTestDataGenerator.tool_name())
return 0
def _cmd_generate_actor_name() -> int:
"""Generate a realistic actor name."""
print(RobotTestDataGenerator.actor_name())
return 0
def _cmd_generate_code() -> int:
"""Generate a realistic Python code snippet."""
print(RobotTestDataGenerator.python_code())
return 0
def _cmd_generate_prose() -> int:
"""Generate realistic Python-like prose."""
print(RobotTestDataGenerator.python_prose())
return 0
def _cmd_generate_file_path() -> int:
"""Generate a realistic file path."""
print(RobotTestDataGenerator.file_path())
return 0
def _cmd_generate_uko_uri() -> int:
"""Generate a realistic UKO node URI."""
print(RobotTestDataGenerator.uko_node_uri())
return 0
def _cmd_generate_resource_uri() -> int:
"""Generate a realistic resource URI."""
print(RobotTestDataGenerator.resource_uri())
return 0
def _cmd_generate_relevance_score() -> int:
"""Generate a realistic relevance score."""
print(RobotTestDataGenerator.relevance_score())
return 0
def _cmd_generate_token_count() -> int:
"""Generate a realistic token count."""
print(RobotTestDataGenerator.token_count())
return 0
def _cmd_generate_detail_depth() -> int:
"""Generate a realistic detail depth."""
print(RobotTestDataGenerator.detail_depth())
return 0
def _cmd_generate_ulid() -> int:
"""Generate a valid ULID."""
print(RobotTestDataGenerator.ulid())
return 0
def _cmd_generate_budget() -> int:
"""Generate a context budget as JSON."""
budget_dict = RobotContextBudgetFactory.create_dict()
print(json.dumps(budget_dict, indent=2))
return 0
# ---------------------------------------------------------------------------
# CLI dispatcher
# ---------------------------------------------------------------------------
_COMMANDS: dict[str, Callable[[], int]] = {
"generate-fragment": _cmd_generate_fragment,
"generate-fragments": _cmd_generate_fragments,
"generate-project-name": _cmd_generate_project_name,
"generate-skill-name": _cmd_generate_skill_name,
"generate-tool-name": _cmd_generate_tool_name,
"generate-actor-name": _cmd_generate_actor_name,
"generate-code": _cmd_generate_code,
"generate-prose": _cmd_generate_prose,
"generate-file-path": _cmd_generate_file_path,
"generate-uko-uri": _cmd_generate_uko_uri,
"generate-resource-uri": _cmd_generate_resource_uri,
"generate-relevance-score": _cmd_generate_relevance_score,
"generate-token-count": _cmd_generate_token_count,
"generate-detail-depth": _cmd_generate_detail_depth,
"generate-ulid": _cmd_generate_ulid,
"generate-budget": _cmd_generate_budget,
}
def main() -> int:
"""Main entry point for CLI."""
if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS:
print(
f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>",
file=sys.stderr,
)
return 1
try:
return _COMMANDS[sys.argv[1]]()
except Exception as exc:
print(f"FAIL: {exc}", file=sys.stderr)
import traceback
traceback.print_exc()
return 1
if __name__ == "__main__":
sys.exit(main())
@@ -16,11 +16,11 @@ def register_registry_agents(
This expects actor_registry to expose list_actors() and get_actor(name).
Each actor is injected as a callable via process_message_sync if present;
otherwise skips.
Exceptions from actor_registry.list_actors() or route_bridge refresh are
allowed to propagate to the caller per CONTRIBUTING.md fail-fast policy.
"""
try:
actors = actor_registry.list_actors() # type: ignore[attr-defined]
except Exception:
return
actors = actor_registry.list_actors() # type: ignore[attr-defined]
for actor in actors or []:
# Prefer synchronous processing if available
@@ -29,7 +29,4 @@ def register_registry_agents(
# RouteBridge caches agents as a dict; refresh it
if hasattr(route_bridge, "agents"):
try:
route_bridge.agents = {a.name: a for a in actors}
except Exception:
route_bridge.agents = {}
route_bridge.agents = {a.name: a for a in actors}