"""Step definitions for semantic validation scenarios. Covers SemanticValidationService, built-in rules, rule registry, cache, severity mapping, config keys, and pipeline integration. """ from __future__ import annotations from typing import TYPE_CHECKING from behave import given, then, when if TYPE_CHECKING: from behave.runner import Context from cleveragents.application.services.semantic_validation_rules import ( APIMisuseRule, BrokenReferenceRule, DependencyCycleRule, DuplicateImportRule, MissingImportRule, MissingSymbolRule, SemanticCheckResult, SemanticValidationSeverity, SyntaxCheckRule, ) from cleveragents.application.services.semantic_validation_service import ( CONFIG_KEY_ENABLED, CONFIG_KEY_PYTHON_ENABLED, CONFIG_KEY_SEVERITY_MAPPING, DEFAULT_CONFIG, SemanticValidationCache, SemanticValidationService, create_default_registry, map_severity_to_mode, resolve_severity, ) # -- Fixtures ---------------------------------------------------------------- VALID_PY = "import os\n\nx = 10\n\ndef hello():\n return x\n" SYNTAX_ERR_PY = "def broken(\n return 42\n" SUSPICIOUS_IMPORT_PY = "import _nonexistent_private_module\n" STD_IMPORT_PY = "import os\nimport sys\nimport json\n" ALL_DEFINED_PY = "import os\n\nx = 10\n\ndef foo():\n return x\n" UNDEF_REF_PY = "def foo():\n return bar_undefined_xyz\n" UNIQUE_REL_PY = "from . import alpha\nfrom . import beta\n" DUP_REL_PY = "from . import alpha\nfrom . import alpha\n" CLEAN_PY = "import json\ndata = json.loads('{}')\n" EVAL_PY = "result = eval('1 + 2')\n" SCOPED_PY = "import os\n\ndef greet(name):\n msg = f'Hi {name}'\n return msg\n" UNDEF_SYM_PY = "def compute(x):\n return x + undefined_thing_xyz\n" FUNC_LOCAL_PY = "def process(items):\n result = []\n for item in items:\n result.append(item)\n return result\n" CLASS_METHOD_PY = ( "class MyClass:\n def method(self):\n return undefined_xyz\n" ) NESTED_FUNC_PY = ( "def outer():\n def inner():\n return 1\n return inner()\n" ) ANNASSIGN_PY = "x: int = 10\n\ndef foo():\n return x\n" WITH_STMT_PY = "import io\ndef read_file():\n with io.open('test.txt') as f:\n return f.read()\n" EXEC_PY = "exec('print(1)')\n" OS_POPEN_PY = "import os\nos.popen('ls')\n" SUBPROCESS_RUN_PY = "import subprocess\nsubprocess.run(['echo', 'hello'])\n" COMPREHENSION_PY = "def summarise(items):\n return [x * 2 for x in items]\n" NON_PY_FILENAME = "config.yaml" EMPTY_PY = "" # -- Fixtures for coverage improvement ---------------------------------------- FROM_PRIVATE_IMPORT_PY = "from _nonexistent_private_mod import something\n" VARARGS_KWARGS_PY = ( "def func(*args, **kwargs):\n return args, kwargs\n\nfunc(1, key=2)\n" ) FROM_IMPORT_BINDINGS_PY = "from os.path import join\n\nresult = join('a', 'b')\n" EXCEPT_HANDLER_PY = "try:\n x = 1\nexcept Exception as exc:\n y = exc\n" TUPLE_STARRED_PY = "a, b = 1, 2\nc, *d = [3, 4, 5]\nresult = a + b + c\n" WITH_NO_AS_PY = "import warnings\nwith warnings.catch_warnings():\n x = 1\n" SAFE_FLAGGED_MODULE_PY = "import os\nresult = os.getcwd()\n" FUNC_ALL_PARAMS_PY = ( "def func(a, /, b, *args, c=1, **kwargs):\n" " return a + b + c + len(args) + len(kwargs)\n" "\nfunc(1, 2, 3, c=4, d=5)\n" ) FUNC_FOR_LOOP_PY = ( "def total(items):\n" " s = 0\n" " for item in items:\n" " s += item\n" " return s\n" ) FUNC_WITH_STMT_PY = ( "def reader(path):\n with open(path) as fh:\n return fh.read()\n" ) FUNC_EXCEPT_PY = ( "def safe_div(a, b):\n" " try:\n" " return a / b\n" " except ZeroDivisionError as err:\n" " return err\n" ) FUNC_LOCAL_IMPORT_PY = "def get_cwd():\n import os\n return os.getcwd()\n" FUNC_LOCAL_FROM_IMPORT_PY = ( "def get_path():\n from os.path import join\n return join('a', 'b')\n" ) # -- Background -------------------------------------------------------------- @given("a semantic validation test environment") def step_env(context: Context) -> None: context.sv_result = None context.sv_results = None context.sv_source = "" context.sv_filename = "test.py" context.sv_service = None context.sv_cache = None context.sv_registry = None context.sv_pipeline_results = None context.sv_normalised = None context.sv_custom_mapping = None # -- Severity enum ----------------------------------------------------------- @then('the semantic severity enum has values "info", "warn", "error"') def step_sev_vals(context: Context) -> None: assert SemanticValidationSeverity.INFO == "info" assert SemanticValidationSeverity.WARN == "warn" assert SemanticValidationSeverity.ERROR == "error" # -- SemanticCheckResult model ----------------------------------------------- @given('a semantic check result with passed true and message "{msg}"') def step_cr_pass(context: Context, msg: str) -> None: context.sv_result = SemanticCheckResult(passed=True, message=msg) @given('a semantic check result with passed false and message "{msg}" and data') def step_cr_fail(context: Context, msg: str) -> None: context.sv_result = SemanticCheckResult( passed=False, message=msg, data={"detail": "x"}, severity=SemanticValidationSeverity.ERROR, ) @then("the semantic check result passed is true") def step_r_true(context: Context) -> None: assert context.sv_result.passed is True @then("the semantic check result passed is false") def step_r_false(context: Context) -> None: assert context.sv_result.passed is False @then('the semantic check result message is "{msg}"') def step_r_msg(context: Context, msg: str) -> None: assert context.sv_result.message == msg @then("the semantic check result data is none") def step_r_none(context: Context) -> None: assert context.sv_result.data is None @then('the semantic check result has data key "{key}"') def step_r_dk(context: Context, key: str) -> None: assert context.sv_result.data is not None and key in context.sv_result.data @then('the semantic check result message contains "{text}"') def step_r_mc(context: Context, text: str) -> None: assert text in context.sv_result.message @then('the semantic check result severity is "{sev}"') def step_r_sev(context: Context, sev: str) -> None: assert context.sv_result.severity.value == sev # -- Source code given steps ------------------------------------------------- @given("valid Python source code") def step_src_valid(context: Context) -> None: context.sv_source = VALID_PY @given("Python source code with a syntax error") def step_src_syn(context: Context) -> None: context.sv_source = SYNTAX_ERR_PY @given("Python source with standard imports") def step_src_std(context: Context) -> None: context.sv_source = STD_IMPORT_PY @given("Python source with suspicious private imports") def step_src_sus(context: Context) -> None: context.sv_source = SUSPICIOUS_IMPORT_PY @given("Python source with all names defined") def step_src_def(context: Context) -> None: context.sv_source = ALL_DEFINED_PY @given("Python source with undefined references") def step_src_uref(context: Context) -> None: context.sv_source = UNDEF_REF_PY @given("Python source with unique relative imports") def step_src_urel(context: Context) -> None: context.sv_source = UNIQUE_REL_PY @given("Python source with duplicate relative imports") def step_src_drel(context: Context) -> None: context.sv_source = DUP_REL_PY @given("Python source without API misuse") def step_src_clean(context: Context) -> None: context.sv_source = CLEAN_PY @given("Python source using eval") def step_src_eval(context: Context) -> None: context.sv_source = EVAL_PY @given("Python source with well-scoped functions") def step_src_scoped(context: Context) -> None: context.sv_source = SCOPED_PY @given("Python source with undefined symbols in functions") def step_src_usym(context: Context) -> None: context.sv_source = UNDEF_SYM_PY @given("Python source with function-local variables") def step_src_func_local(context: Context) -> None: context.sv_source = FUNC_LOCAL_PY @given("Python source with a class method using undefined symbol") def step_src_cls_method(context: Context) -> None: context.sv_source = CLASS_METHOD_PY @given("Python source with nested functions") def step_src_nested(context: Context) -> None: context.sv_source = NESTED_FUNC_PY @given("Python source with annotated assignment") def step_src_annassign(context: Context) -> None: context.sv_source = ANNASSIGN_PY @given("Python source with with-statement variable") def step_src_with_stmt(context: Context) -> None: context.sv_source = WITH_STMT_PY @given("Python source using exec") def step_src_exec(context: Context) -> None: context.sv_source = EXEC_PY @given("Python source using os.popen") def step_src_os_popen(context: Context) -> None: context.sv_source = OS_POPEN_PY @given("Python source using subprocess.run") def step_src_subprocess_run(context: Context) -> None: context.sv_source = SUBPROCESS_RUN_PY @given("Python source with comprehension variables") def step_src_comprehension(context: Context) -> None: context.sv_source = COMPREHENSION_PY @given("empty Python source code") def step_src_empty(context: Context) -> None: context.sv_source = EMPTY_PY @given("Python source with eval in a string literal") def step_src_eval_str(context: Context) -> None: context.sv_source = 'description = "Do not use eval() in production"\n' @given("a non-Python filename") def step_non_py_filename(context: Context) -> None: context.sv_filename = NON_PY_FILENAME # -- Rule execution ---------------------------------------------------------- @when("the syntax check rule runs") def step_run_syn(context: Context) -> None: context.sv_result = SyntaxCheckRule().check(context.sv_source, context.sv_filename) @when("the missing import rule runs") def step_run_imp(context: Context) -> None: context.sv_result = MissingImportRule().check( context.sv_source, context.sv_filename ) @when("the broken reference rule runs") def step_run_ref(context: Context) -> None: context.sv_result = BrokenReferenceRule().check( context.sv_source, context.sv_filename ) @when("the dependency cycle rule runs") def step_run_cyc(context: Context) -> None: context.sv_result = DependencyCycleRule().check( context.sv_source, context.sv_filename ) @when("the API misuse rule runs") def step_run_api(context: Context) -> None: context.sv_result = APIMisuseRule().check(context.sv_source, context.sv_filename) @when("the missing symbol rule runs") def step_run_sym(context: Context) -> None: context.sv_result = MissingSymbolRule().check( context.sv_source, context.sv_filename ) @when("the duplicate import rule runs") def step_run_dup(context: Context) -> None: context.sv_result = DuplicateImportRule().check( context.sv_source, context.sv_filename ) # -- Registry ---------------------------------------------------------------- @given("a default semantic rule registry") def step_reg(context: Context) -> None: context.sv_registry = create_default_registry() @then("the registry has {count:d} rules") def step_reg_cnt(context: Context, count: int) -> None: assert len(context.sv_registry) == count @then('the registry contains rule "{name}"') def step_reg_has(context: Context, name: str) -> None: assert context.sv_registry.get(name) is not None @then('the registry returns None for rule "{name}"') def step_reg_none(context: Context, name: str) -> None: assert context.sv_registry.get(name) is None @when('the rule "{name}" is removed from the registry') def step_reg_rm(context: Context, name: str) -> None: context.sv_registry.remove(name) @then('removing rule "{name}" returns false') def step_reg_rm_f(context: Context, name: str) -> None: assert context.sv_registry.remove(name) is False # -- Cache ------------------------------------------------------------------- @given("a semantic validation cache") def step_cache(context: Context) -> None: context.sv_cache = SemanticValidationCache() @given("a semantic validation cache with max size {n:d}") def step_cache_max(context: Context, n: int) -> None: context.sv_cache = SemanticValidationCache(max_size=n) @given('a cached semantic result for rule "{rule}" and hash "{h}"') def step_cache_put(context: Context, rule: str, h: str) -> None: context.sv_cache.put(rule, h, SemanticCheckResult(passed=True, message="cached")) @then('the cache returns None for rule "{rule}" and hash "{h}"') def step_cache_miss(context: Context, rule: str, h: str) -> None: assert context.sv_cache.get(rule, h) is None @then('the cache returns the result for rule "{rule}" and hash "{h}"') def step_cache_hit(context: Context, rule: str, h: str) -> None: r = context.sv_cache.get(rule, h) assert r is not None and r.message == "cached" @when('the cache entry for rule "{rule}" and hash "{h}" is invalidated') def step_cache_inv(context: Context, rule: str, h: str) -> None: context.sv_cache.invalidate(rule, h) @when("the semantic cache is cleared") def step_cache_clr(context: Context) -> None: context.sv_cache.clear() @then("the semantic cache has {count:d} entries") def step_cache_cnt(context: Context, count: int) -> None: assert len(context.sv_cache) == count @then('the semantic cache hash for "{text}" is consistent') def step_cache_hash(context: Context, text: str) -> None: h1 = SemanticValidationCache.compute_hash(text) assert h1 == SemanticValidationCache.compute_hash(text) and len(h1) == 64 # -- Severity mapping -------------------------------------------------------- @then('semantic severity "{sev}" maps to validation mode "{mode}"') def step_sev_map(context: Context, sev: str, mode: str) -> None: assert map_severity_to_mode(SemanticValidationSeverity(sev)).value == mode @then('resolve_severity for "{rule}" returns "{expected}"') def step_resolve(context: Context, rule: str, expected: str) -> None: assert resolve_severity(rule).value == expected @given('a custom severity mapping with "{rule}" as "{sev}"') def step_cust_map(context: Context, rule: str, sev: str) -> None: context.sv_custom_mapping = {rule: sev} @then('resolve_severity with custom mapping for "{rule}" returns "{expected}"') def step_resolve_c(context: Context, rule: str, expected: str) -> None: assert resolve_severity(rule, context.sv_custom_mapping).value == expected # -- Service ----------------------------------------------------------------- @given("a semantic validation service") def step_svc(context: Context) -> None: context.sv_service = SemanticValidationService() @given("a semantic validation service with semantic disabled") def step_svc_off(context: Context) -> None: context.sv_service = SemanticValidationService(config={CONFIG_KEY_ENABLED: False}) @given("a semantic validation service with python disabled") def step_svc_pyoff(context: Context) -> None: context.sv_service = SemanticValidationService( config={CONFIG_KEY_PYTHON_ENABLED: False}, ) @when("the service checks the file") def step_svc_chk(context: Context) -> None: context.sv_results = context.sv_service.check_file( context.sv_source, context.sv_filename, ) @when("the service checks the python file") def step_svc_chk_py(context: Context) -> None: context.sv_results = context.sv_service.check_file(context.sv_source, "module.py") @when('the service checks the file with rules "{rules}"') def step_svc_chk_r(context: Context, rules: str) -> None: context.sv_results = context.sv_service.check_file( context.sv_source, context.sv_filename, rule_names=[r.strip() for r in rules.split(",")], ) @when("the service checks the file twice") def step_svc_chk2(context: Context) -> None: context.sv_service.check_file(context.sv_source, context.sv_filename) context.sv_results = context.sv_service.check_file( context.sv_source, context.sv_filename, ) @when("the service checks the non-python file") def step_svc_chk_nonpy(context: Context) -> None: context.sv_filename = NON_PY_FILENAME context.sv_results = context.sv_service.check_file( context.sv_source, context.sv_filename, ) @then("all semantic check results passed") def step_all_pass(context: Context) -> None: assert all(r.passed for r in context.sv_results) @then("at least one semantic check result failed") def step_some_fail(context: Context) -> None: assert any(not r.passed for r in context.sv_results) @then('a failing result has message containing "{text}"') def step_fail_msg(context: Context, text: str) -> None: assert any(text in r.message for r in context.sv_results if not r.passed) @then("only {count:d} semantic check result is returned") def step_cnt1(context: Context, count: int) -> None: assert len(context.sv_results) == count @then("{count:d} semantic check results are returned") def step_cnt(context: Context, count: int) -> None: assert len(context.sv_results) == count @then("the cache has entries") def step_cache_has(context: Context) -> None: assert len(context.sv_service.cache) > 0 # -- Pipeline integration --------------------------------------------------- @when("the service returns pipeline results") def step_pipe(context: Context) -> None: context.sv_pipeline_results = context.sv_service.as_pipeline_results( context.sv_source, context.sv_filename, ) @then('each pipeline result has keys "passed" and "message" and "data"') def step_pipe_k(context: Context) -> None: for pr in context.sv_pipeline_results: assert "passed" in pr and "message" in pr and "data" in pr @then('each pipeline result has key "{key}"') def step_pipe_k1(context: Context, key: str) -> None: for pr in context.sv_pipeline_results: assert key in pr @when("the result is normalised") def step_norm(context: Context) -> None: context.sv_normalised = SemanticValidationService().normalise_output( context.sv_result ) @then('the normalised output has keys "passed" and "message" and "data"') def step_norm_k(context: Context) -> None: n = context.sv_normalised assert "passed" in n and "message" in n and "data" in n # -- Config keys ------------------------------------------------------------- @then('config key "{key}" exists') def step_cfg(context: Context, key: str) -> None: assert key in DEFAULT_CONFIG # -- Required vs informational ----------------------------------------------- @then('a pipeline result with severity "{sev}" has mode "{mode}"') def step_pipe_mode(context: Context, sev: str, mode: str) -> None: matching = [r for r in context.sv_pipeline_results if r["severity"] == sev] assert len(matching) > 0 assert all(r["mode"] == mode for r in matching) # -- Safe initialisation and cleanup ----------------------------------------- @then("creating a semantic validation cache with max size 0 raises ValueError") def step_cache_invalid_size(context: Context) -> None: raised = False try: SemanticValidationCache(max_size=0) except ValueError: raised = True assert raised, "Expected ValueError for max_size=0" @when( 'the cache entry for rule "{rule}" and hash "{h}" is overwritten ' 'with message "{msg}"' ) def step_cache_overwrite(context: Context, rule: str, h: str, msg: str) -> None: context.sv_cache.put(rule, h, SemanticCheckResult(passed=True, message=msg)) @then( 'the cache returns the updated result for rule "{rule}" and hash "{h}" ' 'with message "{msg}"' ) def step_cache_updated(context: Context, rule: str, h: str, msg: str) -> None: r = context.sv_cache.get(rule, h) assert r is not None and r.message == msg @then("the registry list_rules returns sorted rule names") def step_reg_list(context: Context) -> None: names = context.sv_registry.list_rules() assert names == sorted(names) assert len(names) == len(context.sv_registry) @then("the service registry property returns a registry with {count:d} rules") def step_svc_reg(context: Context, count: int) -> None: assert len(context.sv_service.registry) == count @given("a semantic validation service with non-dict severity mapping") def step_svc_nondict_sev(context: Context) -> None: context.sv_service = SemanticValidationService( config={CONFIG_KEY_SEVERITY_MAPPING: "not-a-dict"}, ) # -- MissingImportRule from-import form -------------------------------------- @given("Python source with from-import of private module") def step_src_from_priv(context: Context) -> None: context.sv_source = FROM_PRIVATE_IMPORT_PY # -- BrokenReferenceRule advanced scope patterns ----------------------------- @given("Python source with varargs and kwargs function") def step_src_varargs(context: Context) -> None: context.sv_source = VARARGS_KWARGS_PY @given("Python source with from-import bindings") def step_src_from_import(context: Context) -> None: context.sv_source = FROM_IMPORT_BINDINGS_PY @given("Python source with except handler variable") def step_src_except(context: Context) -> None: context.sv_source = EXCEPT_HANDLER_PY @given("Python source with tuple and starred unpacking") def step_src_tuple_star(context: Context) -> None: context.sv_source = TUPLE_STARRED_PY @given("Python source with with-statement without as clause") def step_src_with_no_as(context: Context) -> None: context.sv_source = WITH_NO_AS_PY # -- APIMisuseRule safe attribute on flagged module -------------------------- @given("Python source calling a safe method on a flagged module") def step_src_safe_flagged(context: Context) -> None: context.sv_source = SAFE_FLAGGED_MODULE_PY # -- MissingSymbolRule function-local binding patterns ----------------------- @given("Python source with function using varargs kwonly and kwargs") def step_src_func_all_params(context: Context) -> None: context.sv_source = FUNC_ALL_PARAMS_PY @given("Python source with function containing for-loop variable") def step_src_func_for(context: Context) -> None: context.sv_source = FUNC_FOR_LOOP_PY @given("Python source with function containing with-statement variable") def step_src_func_with(context: Context) -> None: context.sv_source = FUNC_WITH_STMT_PY @given("Python source with function containing except handler variable") def step_src_func_except(context: Context) -> None: context.sv_source = FUNC_EXCEPT_PY @given("Python source with function containing local import") def step_src_func_import(context: Context) -> None: context.sv_source = FUNC_LOCAL_IMPORT_PY @given("Python source with function containing local from-import") def step_src_func_from_import(context: Context) -> None: context.sv_source = FUNC_LOCAL_FROM_IMPORT_PY