cleveragents-core/robot/helper_semantic_validation.py

"""Helper script for Robot Framework semantic validation tests.

Self-contained Python helper that exercises the SemanticValidationService
and related components, printing sentinel strings on success and exiting
with code 1 on failure.
"""

from __future__ import annotations

import sys
from collections.abc import Callable
from pathlib import Path

# Ensure src is importable when run from workspace root
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from cleveragents.application.services.semantic_validation_rules import (
    APIMisuseRule,
    BrokenReferenceRule,
    DuplicateImportRule,
    MissingSymbolRule,
    SemanticCheckResult,
    SemanticValidationSeverity,
    SyntaxCheckRule,
)
from cleveragents.application.services.semantic_validation_service import (
    CONFIG_KEY_ENABLED,
    SemanticValidationCache,
    SemanticValidationService,
    create_default_registry,
    map_severity_to_mode,
)
from cleveragents.domain.models.core.tool import ValidationMode

# ---------------------------------------------------------------------------
# Source code fixtures
# ---------------------------------------------------------------------------

VALID_PYTHON = """\
import os

x = 10

def hello():
    return x
"""

SYNTAX_ERROR_PYTHON = """\
def broken(
    return 42
"""

EVAL_PYTHON = """\
result = eval("1 + 2")
"""

FUNC_LOCAL_PYTHON = """\
def process(items):
    result = []
    for item in items:
        result.append(item)
    return result
"""

CLASS_METHOD_PYTHON = """\
class MyClass:
    def method(self):
        return undefined_xyz
"""


# ---------------------------------------------------------------------------
# Sub-commands
# ---------------------------------------------------------------------------


def test_models() -> None:
    """Verify SemanticCheckResult and SemanticValidationSeverity."""
    result = SemanticCheckResult(
        passed=True,
        message="All good",
        severity=SemanticValidationSeverity.INFO,
    )
    assert result.passed is True
    assert result.message == "All good"
    assert result.severity == SemanticValidationSeverity.INFO
    assert result.data is None

    assert SemanticValidationSeverity.INFO == "info"
    assert SemanticValidationSeverity.WARN == "warn"
    assert SemanticValidationSeverity.ERROR == "error"
    print("models-ok")


def test_syntax_pass() -> None:
    """SyntaxCheckRule passes for valid Python."""
    rule = SyntaxCheckRule()
    result = rule.check(VALID_PYTHON, "test.py")
    assert result.passed is True
    print("syntax-pass-ok")


def test_syntax_fail() -> None:
    """SyntaxCheckRule fails for invalid Python."""
    rule = SyntaxCheckRule()
    result = rule.check(SYNTAX_ERROR_PYTHON, "test.py")
    assert result.passed is False
    assert "Syntax error" in result.message
    print("syntax-fail-ok")


def test_registry() -> None:
    """Default registry has all six built-in rules."""
    registry = create_default_registry()
    assert len(registry) == 6
    names = registry.list_rules()
    assert "syntax_error" in names
    assert "missing_import" in names
    assert "broken_reference" in names
    assert "duplicate_import" in names
    assert "api_misuse" in names
    assert "missing_symbol" in names
    print("registry-ok")


def test_cache() -> None:
    """Cache stores and retrieves correctly."""
    cache = SemanticValidationCache()
    assert cache.get("rule", "hash1") is None

    result = SemanticCheckResult(passed=True, message="cached")
    cache.put("rule", "hash1", result)
    cached = cache.get("rule", "hash1")
    assert cached is not None
    assert cached.message == "cached"

    cache.invalidate("rule", "hash1")
    assert cache.get("rule", "hash1") is None

    h1 = SemanticValidationCache.compute_hash("hello")
    h2 = SemanticValidationCache.compute_hash("hello")
    assert h1 == h2
    assert len(h1) == 64
    print("cache-ok")


def test_service_check() -> None:
    """Service runs all rules."""
    service = SemanticValidationService()
    results = service.check_file(VALID_PYTHON, "test.py")
    assert len(results) > 0
    assert all(r.passed for r in results)
    print("service-check-ok")


def test_severity_mapping() -> None:
    """Severity maps to correct validation mode."""
    assert (
        map_severity_to_mode(SemanticValidationSeverity.ERROR)
        == ValidationMode.REQUIRED
    )
    assert (
        map_severity_to_mode(SemanticValidationSeverity.WARN)
        == ValidationMode.INFORMATIONAL
    )
    assert (
        map_severity_to_mode(SemanticValidationSeverity.INFO)
        == ValidationMode.INFORMATIONAL
    )
    print("severity-mapping-ok")


def test_pipeline_integration() -> None:
    """as_pipeline_results returns dicts with expected keys."""
    service = SemanticValidationService()
    results = service.as_pipeline_results(VALID_PYTHON, "test.py")
    assert len(results) > 0
    for r in results:
        assert "passed" in r
        assert "message" in r
        assert "data" in r
        assert "severity" in r
        assert "mode" in r
    print("pipeline-integration-ok")


def test_config_disabled() -> None:
    """Service returns empty when disabled."""
    service = SemanticValidationService(config={CONFIG_KEY_ENABLED: False})
    results = service.check_file(VALID_PYTHON, "test.py")
    assert len(results) == 0
    print("config-disabled-ok")


def test_api_misuse() -> None:
    """APIMisuseRule detects eval usage via AST analysis."""
    rule = APIMisuseRule()
    result = rule.check(EVAL_PYTHON, "test.py")
    assert result.passed is False
    assert "API misuse" in result.message
    print("api-misuse-ok")


def test_broken_ref_scope() -> None:
    """BrokenReferenceRule does not false-positive on function locals."""
    rule = BrokenReferenceRule()
    result = rule.check(FUNC_LOCAL_PYTHON, "test.py")
    assert result.passed is True, f"Expected pass, got: {result.message}"
    print("broken-ref-scope-ok")


def test_missing_symbol_class() -> None:
    """MissingSymbolRule detects undefined symbols in class methods."""
    rule = MissingSymbolRule()
    result = rule.check(CLASS_METHOD_PYTHON, "test.py")
    assert result.passed is False, f"Expected fail, got: {result.message}"
    assert "missing symbol" in result.message
    print("missing-symbol-class-ok")


def test_cache_lru() -> None:
    """Cache evicts LRU entries when max_size is exceeded."""
    cache = SemanticValidationCache(max_size=2)
    r = SemanticCheckResult(passed=True, message="ok")
    cache.put("a", "h1", r)
    cache.put("b", "h2", r)
    cache.put("c", "h3", r)
    assert len(cache) == 2
    assert cache.get("a", "h1") is None  # evicted
    assert cache.get("b", "h2") is not None
    assert cache.get("c", "h3") is not None
    print("cache-lru-ok")


def test_non_python_skip() -> None:
    """Service returns empty for non-Python files."""
    service = SemanticValidationService()
    results = service.check_file(VALID_PYTHON, "config.yaml")
    assert len(results) == 0
    print("non-python-skip-ok")


def test_duplicate_import_alias() -> None:
    """DuplicateImportRule is accessible via DependencyCycleRule alias."""
    rule = DuplicateImportRule()
    assert rule.name == "duplicate_import"
    print("duplicate-import-alias-ok")


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

_COMMANDS: dict[str, Callable[[], None]] = {
    "models": test_models,
    "syntax_pass": test_syntax_pass,
    "syntax_fail": test_syntax_fail,
    "registry": test_registry,
    "cache": test_cache,
    "service_check": test_service_check,
    "severity_mapping": test_severity_mapping,
    "pipeline_integration": test_pipeline_integration,
    "config_disabled": test_config_disabled,
    "api_misuse": test_api_misuse,
    "broken_ref_scope": test_broken_ref_scope,
    "missing_symbol_class": test_missing_symbol_class,
    "cache_lru": test_cache_lru,
    "non_python_skip": test_non_python_skip,
    "duplicate_import_alias": test_duplicate_import_alias,
}

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: helper_semantic_validation.py <command>", file=sys.stderr)
        sys.exit(1)

    cmd = sys.argv[1]
    if cmd not in _COMMANDS:
        print(f"Unknown command: {cmd}", file=sys.stderr)
        sys.exit(1)

    try:
        _COMMANDS[cmd]()
    except Exception as exc:
        print(f"FAILED: {exc}", file=sys.stderr)
        sys.exit(1)