"""Helper script for Robot Framework semantic validation tests. Self-contained Python helper that exercises the SemanticValidationService and related components, printing sentinel strings on success and exiting with code 1 on failure. """ from __future__ import annotations import sys from collections.abc import Callable from pathlib import Path # Ensure src is importable when run from workspace root sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) from cleveragents.application.services.semantic_validation_rules import ( APIMisuseRule, BrokenReferenceRule, DuplicateImportRule, MissingSymbolRule, SemanticCheckResult, SemanticValidationSeverity, SyntaxCheckRule, ) from cleveragents.application.services.semantic_validation_service import ( CONFIG_KEY_ENABLED, SemanticValidationCache, SemanticValidationService, create_default_registry, map_severity_to_mode, ) from cleveragents.domain.models.core.tool import ValidationMode # --------------------------------------------------------------------------- # Source code fixtures # --------------------------------------------------------------------------- VALID_PYTHON = """\ import os x = 10 def hello(): return x """ SYNTAX_ERROR_PYTHON = """\ def broken( return 42 """ EVAL_PYTHON = """\ result = eval("1 + 2") """ FUNC_LOCAL_PYTHON = """\ def process(items): result = [] for item in items: result.append(item) return result """ CLASS_METHOD_PYTHON = """\ class MyClass: def method(self): return undefined_xyz """ # --------------------------------------------------------------------------- # Sub-commands # --------------------------------------------------------------------------- def test_models() -> None: """Verify SemanticCheckResult and SemanticValidationSeverity.""" result = SemanticCheckResult( passed=True, message="All good", severity=SemanticValidationSeverity.INFO, ) assert result.passed is True assert result.message == "All good" assert result.severity == SemanticValidationSeverity.INFO assert result.data is None assert SemanticValidationSeverity.INFO == "info" assert SemanticValidationSeverity.WARN == "warn" assert SemanticValidationSeverity.ERROR == "error" print("models-ok") def test_syntax_pass() -> None: """SyntaxCheckRule passes for valid Python.""" rule = SyntaxCheckRule() result = rule.check(VALID_PYTHON, "test.py") assert result.passed is True print("syntax-pass-ok") def test_syntax_fail() -> None: """SyntaxCheckRule fails for invalid Python.""" rule = SyntaxCheckRule() result = rule.check(SYNTAX_ERROR_PYTHON, "test.py") assert result.passed is False assert "Syntax error" in result.message print("syntax-fail-ok") def test_registry() -> None: """Default registry has all six built-in rules.""" registry = create_default_registry() assert len(registry) == 6 names = registry.list_rules() assert "syntax_error" in names assert "missing_import" in names assert "broken_reference" in names assert "duplicate_import" in names assert "api_misuse" in names assert "missing_symbol" in names print("registry-ok") def test_cache() -> None: """Cache stores and retrieves correctly.""" cache = SemanticValidationCache() assert cache.get("rule", "hash1") is None result = SemanticCheckResult(passed=True, message="cached") cache.put("rule", "hash1", result) cached = cache.get("rule", "hash1") assert cached is not None assert cached.message == "cached" cache.invalidate("rule", "hash1") assert cache.get("rule", "hash1") is None h1 = SemanticValidationCache.compute_hash("hello") h2 = SemanticValidationCache.compute_hash("hello") assert h1 == h2 assert len(h1) == 64 print("cache-ok") def test_service_check() -> None: """Service runs all rules.""" service = SemanticValidationService() results = service.check_file(VALID_PYTHON, "test.py") assert len(results) > 0 assert all(r.passed for r in results) print("service-check-ok") def test_severity_mapping() -> None: """Severity maps to correct validation mode.""" assert ( map_severity_to_mode(SemanticValidationSeverity.ERROR) == ValidationMode.REQUIRED ) assert ( map_severity_to_mode(SemanticValidationSeverity.WARN) == ValidationMode.INFORMATIONAL ) assert ( map_severity_to_mode(SemanticValidationSeverity.INFO) == ValidationMode.INFORMATIONAL ) print("severity-mapping-ok") def test_pipeline_integration() -> None: """as_pipeline_results returns dicts with expected keys.""" service = SemanticValidationService() results = service.as_pipeline_results(VALID_PYTHON, "test.py") assert len(results) > 0 for r in results: assert "passed" in r assert "message" in r assert "data" in r assert "severity" in r assert "mode" in r print("pipeline-integration-ok") def test_config_disabled() -> None: """Service returns empty when disabled.""" service = SemanticValidationService(config={CONFIG_KEY_ENABLED: False}) results = service.check_file(VALID_PYTHON, "test.py") assert len(results) == 0 print("config-disabled-ok") def test_api_misuse() -> None: """APIMisuseRule detects eval usage via AST analysis.""" rule = APIMisuseRule() result = rule.check(EVAL_PYTHON, "test.py") assert result.passed is False assert "API misuse" in result.message print("api-misuse-ok") def test_broken_ref_scope() -> None: """BrokenReferenceRule does not false-positive on function locals.""" rule = BrokenReferenceRule() result = rule.check(FUNC_LOCAL_PYTHON, "test.py") assert result.passed is True, f"Expected pass, got: {result.message}" print("broken-ref-scope-ok") def test_missing_symbol_class() -> None: """MissingSymbolRule detects undefined symbols in class methods.""" rule = MissingSymbolRule() result = rule.check(CLASS_METHOD_PYTHON, "test.py") assert result.passed is False, f"Expected fail, got: {result.message}" assert "missing symbol" in result.message print("missing-symbol-class-ok") def test_cache_lru() -> None: """Cache evicts LRU entries when max_size is exceeded.""" cache = SemanticValidationCache(max_size=2) r = SemanticCheckResult(passed=True, message="ok") cache.put("a", "h1", r) cache.put("b", "h2", r) cache.put("c", "h3", r) assert len(cache) == 2 assert cache.get("a", "h1") is None # evicted assert cache.get("b", "h2") is not None assert cache.get("c", "h3") is not None print("cache-lru-ok") def test_non_python_skip() -> None: """Service returns empty for non-Python files.""" service = SemanticValidationService() results = service.check_file(VALID_PYTHON, "config.yaml") assert len(results) == 0 print("non-python-skip-ok") def test_duplicate_import_alias() -> None: """DuplicateImportRule is accessible via DependencyCycleRule alias.""" rule = DuplicateImportRule() assert rule.name == "duplicate_import" print("duplicate-import-alias-ok") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- _COMMANDS: dict[str, Callable[[], None]] = { "models": test_models, "syntax_pass": test_syntax_pass, "syntax_fail": test_syntax_fail, "registry": test_registry, "cache": test_cache, "service_check": test_service_check, "severity_mapping": test_severity_mapping, "pipeline_integration": test_pipeline_integration, "config_disabled": test_config_disabled, "api_misuse": test_api_misuse, "broken_ref_scope": test_broken_ref_scope, "missing_symbol_class": test_missing_symbol_class, "cache_lru": test_cache_lru, "non_python_skip": test_non_python_skip, "duplicate_import_alias": test_duplicate_import_alias, } if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: helper_semantic_validation.py ", file=sys.stderr) sys.exit(1) cmd = sys.argv[1] if cmd not in _COMMANDS: print(f"Unknown command: {cmd}", file=sys.stderr) sys.exit(1) try: _COMMANDS[cmd]() except Exception as exc: print(f"FAILED: {exc}", file=sys.stderr) sys.exit(1)