diff --git a/CHANGELOG.md b/CHANGELOG.md index e2bb52577..c7182b68f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +- Added `PostgreSQLAnalyzer` and `DockerComposeAnalyzer` domain-specific analyzers + (Phase 2 of issue #588). `PostgreSQLAnalyzer` parses DDL content via regex and + extracts `uko-data:Table`, `uko-data:Column`, `uko-data:ForeignKey`, `uko-data:View`, + and `uko-data:Schema` triples with column metadata (data type, nullability, primary + key). `DockerComposeAnalyzer` parses Docker Compose YAML via `yaml.safe_load` and + extracts `uko-infra:DeploymentUnit`, `uko-infra:Service`, `uko-infra:Port`, + `uko-infra:EnvironmentVariable`, and `uko-infra:connectsTo` triples. Both satisfy + `AnalyzerProtocol` and register in `AnalyzerRegistry` by file extension. Includes + 34 Behave BDD scenarios covering all four analyzers (protocol conformance, registry + operations, triple extraction, error handling, cross-analyzer URI scheme and + confidence checks), 6 Robot Framework integration smoke tests, and updated + `__init__.py` exports. (#588) - Added general-purpose domain event system under `cleveragents.infrastructure.events`. `EventType` StrEnum defines 38 typed event identifiers across 9 domains (plan lifecycle, decision, invariant, actor, diff --git a/features/docker_compose_analyzer.feature b/features/docker_compose_analyzer.feature new file mode 100644 index 000000000..bbbe36174 --- /dev/null +++ b/features/docker_compose_analyzer.feature @@ -0,0 +1,199 @@ +@phase2 @acms @analyzer @docker_compose_analyzer +Feature: DockerComposeAnalyzer + As a CleverAgents developer + I want a Docker Compose analyzer that produces UKO triples from YAML files + So that the ACMS can index infrastructure definitions into the UKO knowledge graph + + Scenario: Analyze services + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + api: + image: node:18 + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:DeploymentUnit" + And the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" + And the triples should include predicate "uko:contains" linking subject "deployment" to object "web" + + Scenario: Analyze ports + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + ports: + - "8080:80" + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" + And the triples should include predicate "uko-infra:exposes" + + Scenario: Analyze environment variables + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + api: + image: node:18 + environment: + DATABASE_URL: postgres://localhost/db + NODE_ENV: production + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" + And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" + And the triples should include predicate "rdfs:label" with object_value "NODE_ENV" + + Scenario: Analyze depends_on + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: + - api + - redis + api: + image: node:18 + redis: + image: redis:7 + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze depends_on as single string + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: db + db: + image: postgres + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze depends_on as dict form + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: + db: + condition: service_healthy + db: + image: postgres + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze volumes + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + volumes: + - ./data:/app/data + """ + Then the triples should include predicate "uko:contains" linking subject "web" to object "volume" + + Scenario: Analyze environment as list form + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + api: + image: node:18 + environment: + - DATABASE_URL=postgres://localhost/db + - NODE_ENV=production + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" + And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" + + Scenario: Analyze service with null body + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + placeholder: + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" + And the triples should include predicate "rdfs:label" with object_value "placeholder" + + Scenario: Analyze dict-form port mapping + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + ports: + - target: 80 + published: 8080 + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" + And the triples should include predicate "rdfs:label" with object_value "8080:80" + + Scenario: Analyze dict-form volume mapping + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + volumes: + - type: bind + source: ./data + target: /data + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:ConfigKey" + And the triples should include predicate "rdfs:label" with object_value "./data:/data" + + Scenario: Analyze empty Docker Compose content raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + Scenario: Analyze whitespace-only Docker Compose content raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing whitespace-only content with this analyzer should raise ValueError + + Scenario: Analyze Docker Compose with empty resource_uri raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing content with empty resource_uri should raise ValueError + + Scenario: Analyze version-only YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + version: "3.8" + """ + Then no triples should be returned + + Scenario: Analyze non-compose YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + name: my-application + settings: + debug: true + log_level: info + """ + Then no triples should be returned + + Scenario: Analyze invalid YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + ports: + - this: is: broken: yaml: [ + """ + Then no triples should be returned diff --git a/features/domain_analyzers.feature b/features/domain_analyzers.feature new file mode 100644 index 000000000..4dfd82120 --- /dev/null +++ b/features/domain_analyzers.feature @@ -0,0 +1,217 @@ +@phase2 @acms @analyzer +Feature: Domain-Specific Analyzers + As a CleverAgents developer + I want domain-specific analyzers that produce UKO triples from resources + So that the ACMS can index code, documents, SQL, and infrastructure into the UKO knowledge graph + + # --------------------------------------------------------------------------- + # AnalyzerProtocol conformance + # --------------------------------------------------------------------------- + + @analyzer_protocol + Scenario: PythonAnalyzer satisfies AnalyzerProtocol + Given a PythonAnalyzer analyzer + Then the analyzer should satisfy the AnalyzerProtocol + And the current analyzer domain should be "python" + + @analyzer_protocol + Scenario: MarkdownAnalyzer satisfies AnalyzerProtocol + Given a MarkdownAnalyzer analyzer + Then the analyzer should satisfy the AnalyzerProtocol + And the current analyzer domain should be "markdown" + + @analyzer_protocol + Scenario: PostgreSQLAnalyzer satisfies AnalyzerProtocol + Given a PostgreSQLAnalyzer analyzer + Then the analyzer should satisfy the AnalyzerProtocol + And the current analyzer domain should be "postgresql" + + @analyzer_protocol + Scenario: DockerComposeAnalyzer satisfies AnalyzerProtocol + Given a DockerComposeAnalyzer analyzer + Then the analyzer should satisfy the AnalyzerProtocol + And the current analyzer domain should be "docker-compose" + + # --------------------------------------------------------------------------- + # AnalyzerRegistry + # --------------------------------------------------------------------------- + + @analyzer_registry + Scenario: Register all 4 analyzers and verify count + Given a fresh analyzer registry + When I register all 4 domain analyzers + Then the registry should contain 4 analyzers + + @analyzer_registry + Scenario: Lookup by extension returns correct analyzer for .py + Given a fresh analyzer registry with all domain analyzers registered + When I look up extension ".py" in the registry + Then the current analyzer domain should be "python" + + @analyzer_registry + Scenario: Lookup by extension returns correct analyzer for .sql + Given a fresh analyzer registry with all domain analyzers registered + When I look up extension ".sql" in the registry + Then the current analyzer domain should be "postgresql" + + @analyzer_registry + Scenario: Lookup for unknown extension returns None + Given a fresh analyzer registry with all domain analyzers registered + When I look up extension ".rs" in the registry + Then the extension lookup result should be None + + @analyzer_registry + Scenario: Register duplicate extension first wins + Given a fresh analyzer registry + When I register a PythonAnalyzer + And I register a duplicate analyzer that also claims ".py" + And I look up extension ".py" in the registry + Then the current analyzer domain should be "python" + + @analyzer_registry + Scenario: list_extensions returns all registered extensions + Given a fresh analyzer registry with all domain analyzers registered + Then the registry extensions should include ".py" + And the registry extensions should include ".pyi" + And the registry extensions should include ".md" + And the registry extensions should include ".markdown" + And the registry extensions should include ".sql" + And the registry extensions should include ".ddl" + And the registry extensions should include ".yml" + And the registry extensions should include ".yaml" + + # --------------------------------------------------------------------------- + # PythonAnalyzer + # --------------------------------------------------------------------------- + + @python_analyzer + Scenario: Analyze Python class definition + Given a PythonAnalyzer analyzer + When I analyze the Python content: + """ + class MyService: + pass + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-code:Module" + And the triples should include predicate "rdf:type" with object_uri "uko-py:Class" + And the triples should include predicate "uko:contains" linking subject "module" to object "MyService" + + @python_analyzer + Scenario: Analyze Python function with docstring + Given a PythonAnalyzer analyzer + When I analyze the Python content: + """ + def process_data(): + '''Process incoming data.''' + pass + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-py:Function" + And the triples should include predicate "uko-doc:hasDocstring" + + @python_analyzer + Scenario: Analyze Python imports + Given a PythonAnalyzer analyzer + When I analyze the Python content: + """ + import os + from pathlib import Path + """ + Then the triples should include predicate "uko:references" with object_uri containing "os" + And the triples should include predicate "uko:references" with object_uri containing "pathlib" + + @python_analyzer + Scenario: Analyze empty Python content raises ValueError + Given a PythonAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + @python_analyzer + Scenario: Analyze invalid Python syntax returns empty list + Given a PythonAnalyzer analyzer + When I analyze the Python content: + """ + def broken( + pass + class ??? + """ + Then no triples should be returned + + # --------------------------------------------------------------------------- + # MarkdownAnalyzer + # --------------------------------------------------------------------------- + + @markdown_analyzer + Scenario: Analyze heading hierarchy + Given a MarkdownAnalyzer analyzer + When I analyze the Markdown content: + """ + # Introduction + Some text. + ## Details + More text. + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-doc:Document" + And the triples should include predicate "rdf:type" with object_uri "uko-doc:Section" + And the triples should include predicate "uko-doc:headingLevel" with object_value "1" + And the triples should include predicate "uko-doc:headingLevel" with object_value "2" + + @markdown_analyzer + Scenario: Analyze code block with language + Given a MarkdownAnalyzer analyzer + When I analyze the Markdown content: + """ + # Example + ```python + print("hello") + ``` + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-code:CodeBlock" + And the triples should include predicate "uko-code:language" with object_value "python" + + @markdown_analyzer + Scenario: Analyze links + Given a MarkdownAnalyzer analyzer + When I analyze the Markdown content: + """ + # Resources + See [Example](https://example.com) for details. + """ + Then the triples should include predicate "uko:references" with object_value "https://example.com" + + @markdown_analyzer + Scenario: Analyze empty Markdown content raises ValueError + Given a MarkdownAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + @markdown_analyzer + Scenario: Analyze content with no headings still produces Document triple + Given a MarkdownAnalyzer analyzer + When I analyze the Markdown content: + """ + Just some plain text without any headings. + Another line of text. + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-doc:Document" + And the triples should not include predicate "rdf:type" with object_uri "uko-doc:Section" + + # --------------------------------------------------------------------------- + # Cross-analyzer + # --------------------------------------------------------------------------- + + @analyzer_cross + Scenario: All analyzers use UKO URI scheme + Given a PythonAnalyzer analyzer + And a MarkdownAnalyzer analyzer + And a PostgreSQLAnalyzer analyzer + And a DockerComposeAnalyzer analyzer + When each analyzer processes its sample content + Then every triple subject_uri should start with "uko://" + + @analyzer_cross + Scenario: Confidence scores default to 1.0 + Given a PythonAnalyzer analyzer + When I analyze the Python content: + """ + class Example: + pass + """ + Then every triple confidence should be 1.0 diff --git a/features/postgresql_analyzer.feature b/features/postgresql_analyzer.feature new file mode 100644 index 000000000..f733e0416 --- /dev/null +++ b/features/postgresql_analyzer.feature @@ -0,0 +1,201 @@ +@phase2 @acms @analyzer @postgresql_analyzer +Feature: PostgreSQLAnalyzer + As a CleverAgents developer + I want a PostgreSQL DDL analyzer that produces UKO triples from SQL files + So that the ACMS can index database schemas into the UKO knowledge graph + + Scenario: Analyze CREATE TABLE + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE users ( + id SERIAL, + name VARCHAR(100) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" + And the triples should include predicate "uko-data:dataType" + + Scenario: Analyze table with NOT NULL and PRIMARY KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE accounts ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) NOT NULL + ); + """ + Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" + And the triples should include predicate "uko-data:isNullable" with object_value "false" + + Scenario: Analyze FOREIGN KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" + And the triples should include predicate "uko-data:foreignKeyTo" + + Scenario: Analyze CREATE VIEW + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE VIEW active_users AS + SELECT * FROM users WHERE active = true; + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" + And the triples should include predicate "rdfs:label" with object_value "active_users" + + Scenario: Analyze CREATE SCHEMA + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE SCHEMA IF NOT EXISTS app; + CREATE TABLE app.users ( + id SERIAL PRIMARY KEY + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Schema" + And the triples should include predicate "rdfs:label" with object_value "app" + And the triples should include predicate "uko:contains" linking subject "schema" to object "users" + + Scenario: Analyze schema-qualified table includes schema in column URIs + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE myschema.items ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" + And the triples should include predicate "uko:contains" linking subject "myschema" to object "items" + + Scenario: Analyze table with composite PRIMARY KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE user_roles ( + user_id INTEGER NOT NULL, + role_id INTEGER NOT NULL, + PRIMARY KEY (user_id, role_id) + ); + """ + Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" + + Scenario: Analyze empty SQL content raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + Scenario: Analyze whitespace-only SQL content raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing whitespace-only content with this analyzer should raise ValueError + + Scenario: Analyze SQL with empty resource_uri raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing content with empty resource_uri should raise ValueError + + Scenario: Analyze SQL with comments does not produce phantom triples + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + -- CREATE TABLE phantom (id INT); + /* CREATE VIEW ghost AS SELECT 1; */ + CREATE TABLE real_table (id SERIAL PRIMARY KEY); + """ + Then the triples should include predicate "rdfs:label" with object_value "real_table" + And the triples should not include predicate "rdfs:label" with object_value "phantom" + And the triples should not include predicate "rdfs:label" with object_value "ghost" + + Scenario: Analyze multi-column FOREIGN KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE user_roles ( + user_id INTEGER NOT NULL, + role_id INTEGER NOT NULL, + PRIMARY KEY (user_id, role_id), + FOREIGN KEY (user_id, role_id) REFERENCES lookup(uid, rid) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" + And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "uid" + And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "rid" + + Scenario: Analyze CREATE VIEW captures viewDefinition + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE VIEW active_users AS + SELECT id, name FROM users WHERE active = true; + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" + And the triples should include predicate "uko-data:viewDefinition" + + Scenario: Duplicate CREATE TABLE is deduplicated + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE items (id SERIAL); + CREATE TABLE items (id SERIAL, name TEXT); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the number of Table type triples should be 1 + + Scenario: Analyze invalid SQL returns empty list + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + THIS IS NOT VALID SQL AT ALL; + JUST RANDOM TEXT HERE; + """ + Then no triples should be returned + + # -- Regression tests for F3 (string-literal parentheses) -- + + Scenario: DEFAULT with parentheses in string literal does not truncate columns + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE config ( + col1 TEXT DEFAULT 'hello(world)', + col2 INTEGER + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "col1" + And the triples should include predicate "rdfs:label" with object_value "col2" + + Scenario: CHECK constraint with string containing parentheses + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE validated ( + status TEXT CHECK (status IN ('active(1)', 'inactive(0)')), + amount INTEGER + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "status" + And the triples should include predicate "rdfs:label" with object_value "amount" + + # -- Regression test for F6 (quoted keyword column names) -- + + Scenario: Quoted keyword column names are not silently dropped + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE reserved_words ( + "primary" INTEGER, + "check" VARCHAR(50), + normal_col TEXT + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "primary" + And the triples should include predicate "rdfs:label" with object_value "check" + And the triples should include predicate "rdfs:label" with object_value "normal_col" diff --git a/features/steps/domain_analyzer_steps.py b/features/steps/domain_analyzer_steps.py new file mode 100644 index 000000000..ee4b3478f --- /dev/null +++ b/features/steps/domain_analyzer_steps.py @@ -0,0 +1,397 @@ +"""Step definitions for domain-specific analyzers feature. + +Covers the four built-in analyzers (Python, Markdown, PostgreSQL, +Docker Compose), the ``AnalyzerRegistry``, and protocol compliance +checks. + +Step patterns are prefixed or worded to avoid collisions with existing +step definitions in ``uko_analyzers_steps.py``. Triple-assertion steps +use ``use_step_matcher("re")`` to avoid ``parse``-format ambiguity. +""" + +from __future__ import annotations + +from behave import given, then, use_step_matcher, when +from behave.runner import Context + +from cleveragents.domain.models.acms.analyzers import ( + AnalyzerProtocol, + AnalyzerRegistry, + UKOTriple, +) +from cleveragents.domain.models.acms.docker_compose_analyzer import ( + DockerComposeAnalyzer, +) +from cleveragents.domain.models.acms.markdown_analyzer import MarkdownAnalyzer +from cleveragents.domain.models.acms.postgresql_analyzer import PostgreSQLAnalyzer +from cleveragents.domain.models.acms.python_analyzer import PythonAnalyzer + +__all__: list[str] = [] + +# --------------------------------------------------------------------------- +# Analyzer type mapping +# --------------------------------------------------------------------------- + +_ANALYZER_CLASSES: dict[str, type] = { + "PythonAnalyzer": PythonAnalyzer, + "MarkdownAnalyzer": MarkdownAnalyzer, + "PostgreSQLAnalyzer": PostgreSQLAnalyzer, + "DockerComposeAnalyzer": DockerComposeAnalyzer, +} + +_ALL_ANALYZERS = ( + PythonAnalyzer, + MarkdownAnalyzer, + PostgreSQLAnalyzer, + DockerComposeAnalyzer, +) + +# Sample content for each analyzer (cross-analyzer scenarios). +_SAMPLE_CONTENT: dict[str, str] = { + "PythonAnalyzer": "class Example:\n pass\n", + "MarkdownAnalyzer": "# Title\nSome text.\n", + "PostgreSQLAnalyzer": ("CREATE TABLE sample (\n id SERIAL PRIMARY KEY\n);\n"), + "DockerComposeAnalyzer": "services:\n web:\n image: nginx\n", +} + +_SAMPLE_URI = "uko://test/resource" + + +# --------------------------------------------------------------------------- +# Given steps +# --------------------------------------------------------------------------- + + +@given("a {analyzer_type} analyzer") +def step_given_named_analyzer(context: Context, analyzer_type: str) -> None: + cls = _ANALYZER_CLASSES.get(analyzer_type) + if cls is None: + raise ValueError( + f"Unknown analyzer type {analyzer_type!r}. " + f"Known types: {sorted(_ANALYZER_CLASSES)}" + ) + context.analyzer = cls() + # Track all created analyzers for cross-analyzer scenarios. + if not hasattr(context, "analyzers"): + context.analyzers = {} + context.analyzers[analyzer_type] = context.analyzer + + +@given("a fresh analyzer registry") +def step_given_fresh_registry(context: Context) -> None: + context.registry = AnalyzerRegistry() + + +@given("a fresh analyzer registry with all domain analyzers registered") +def step_given_fresh_registry_with_all(context: Context) -> None: + context.registry = AnalyzerRegistry() + for cls in _ALL_ANALYZERS: + context.registry.register(cls()) + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when("I register all 4 domain analyzers") +def step_when_register_all_four(context: Context) -> None: + for cls in _ALL_ANALYZERS: + context.registry.register(cls()) + + +@when("I register a PythonAnalyzer") +def step_when_register_python(context: Context) -> None: + context.registry.register(PythonAnalyzer()) + + +@when('I register a duplicate analyzer that also claims ".py"') +def step_when_register_duplicate_py(context: Context) -> None: + """Register a second analyzer whose supported_extensions include .py.""" + + class _DuplicatePyAnalyzer: + @property + def supported_extensions(self) -> frozenset[str]: + return frozenset({".py"}) + + @property + def domain(self) -> str: + return "duplicate" + + def analyze(self, content: str, resource_uri: str) -> list[UKOTriple]: + return [] # pragma: no cover + + _dup: type[AnalyzerProtocol] = _DuplicatePyAnalyzer + del _dup # only used for static type-checking assertion + context.registry.register(_DuplicatePyAnalyzer()) + + +@when('I look up extension "{ext}" in the registry') +def step_when_lookup_extension_in_registry(context: Context, ext: str) -> None: + result = context.registry.get_for_extension(ext) + context.lookup_result = result + if result is not None: + context.analyzer = result + + +@when("I analyze the Python content:") +def step_when_analyze_python(context: Context) -> None: + context.triples = context.analyzer.analyze(context.text, _SAMPLE_URI) + + +@when("I analyze the Markdown content:") +def step_when_analyze_markdown(context: Context) -> None: + context.triples = context.analyzer.analyze(context.text, _SAMPLE_URI) + + +@when("I analyze the SQL content:") +def step_when_analyze_sql(context: Context) -> None: + context.triples = context.analyzer.analyze(context.text, _SAMPLE_URI) + + +@when("I analyze the Docker Compose content:") +def step_when_analyze_docker_compose(context: Context) -> None: + context.triples = context.analyzer.analyze(context.text, _SAMPLE_URI) + + +@when("each analyzer processes its sample content") +def step_when_each_analyzer_processes_sample(context: Context) -> None: + all_triples: list[UKOTriple] = [] + for name, analyzer in context.analyzers.items(): + sample = _SAMPLE_CONTENT[name] + triples = analyzer.analyze(sample, _SAMPLE_URI) + all_triples.extend(triples) + context.triples = all_triples + + +# --------------------------------------------------------------------------- +# Then steps — protocol compliance +# --------------------------------------------------------------------------- + + +@then("the analyzer should satisfy the AnalyzerProtocol") +def step_then_satisfies_protocol(context: Context) -> None: + assert isinstance(context.analyzer, AnalyzerProtocol), ( + f"{type(context.analyzer).__name__} does not satisfy AnalyzerProtocol" + ) + + +@then('the current analyzer domain should be "{domain}"') +def step_then_current_analyzer_domain(context: Context, domain: str) -> None: + actual = context.analyzer.domain + assert actual == domain, f"Expected domain={domain!r}, got {actual!r}" + + +# --------------------------------------------------------------------------- +# Then steps — registry assertions +# --------------------------------------------------------------------------- + + +@then("the registry should contain {count:d} analyzers") +def step_then_registry_count(context: Context, count: int) -> None: + actual = len(context.registry) + assert actual == count, f"Expected {count} analyzers in registry, got {actual}" + + +@then("the extension lookup result should be None") +def step_then_extension_lookup_none(context: Context) -> None: + assert context.lookup_result is None, ( + f"Expected None, got {context.lookup_result!r}" + ) + + +@then('the registry extensions should include "{ext}"') +def step_then_registry_extensions_include(context: Context, ext: str) -> None: + extensions = context.registry.list_extensions() + assert ext in extensions, ( + f"Extension {ext!r} not in registered extensions {extensions}" + ) + + +# --------------------------------------------------------------------------- +# Then steps — triple count assertions +# --------------------------------------------------------------------------- + + +@then("no triples should be returned") +def step_then_no_triples(context: Context) -> None: + actual = len(context.triples) + assert actual == 0, f"Expected 0 triples, got {actual}.\nTriples: {context.triples}" + + +@then("the number of Table type triples should be {count:d}") +def step_then_table_type_count(context: Context, count: int) -> None: + actual = sum( + 1 + for t in context.triples + if t.predicate == "rdf:type" and t.object_uri == "uko-data:Table" + ) + assert actual == count, f"Expected {count} Table type triples, got {actual}" + + +# ========================================================================= +# Switch to regex matcher for triple-assertion steps to avoid ambiguity +# that behave's ``parse`` format causes when patterns share a prefix +# (e.g. "{pred}" greedily consuming "… and object_uri …"). +# ========================================================================= +use_step_matcher("re") + + +@then( + r'the triples should include predicate "(?P[^"]+)"' + r' with object_uri "(?P[^"]+)"' +) +def step_then_triple_pred_obj_uri(context: Context, pred: str, obj: str) -> None: + found = any(t.predicate == pred and t.object_uri == obj for t in context.triples) + assert found, ( + f"No triple with predicate={pred!r} object_uri={obj!r} in {context.triples}" + ) + + +@then( + r'the triples should include predicate "(?P[^"]+)"' + r' with object_value "(?P[^"]+)"' +) +def step_then_triple_pred_obj_val(context: Context, pred: str, val: str) -> None: + found = any(t.predicate == pred and t.object_value == val for t in context.triples) + assert found, ( + f"No triple with predicate={pred!r} object_value={val!r} in {context.triples}" + ) + + +@then( + r'the triples should include predicate "(?P[^"]+)"' + r' with object_uri containing "(?P[^"]+)"' +) +def step_then_triple_pred_obj_uri_contains( + context: Context, pred: str, fragment: str +) -> None: + found = any( + t.predicate == pred and fragment in t.object_uri for t in context.triples + ) + assert found, ( + f"No triple with predicate={pred!r} and object_uri containing " + f"{fragment!r} in {context.triples}" + ) + + +@then( + r'the triples should include predicate "(?P[^"]+)"' + r' linking subject "(?P[^"]+)"' + r' to object "(?P[^"]+)"' +) +def step_then_triple_links_subject_to_object( + context: Context, pred: str, s_frag: str, o_frag: str +) -> None: + found = any( + t.predicate == pred + and s_frag in t.subject_uri + and (o_frag in t.object_uri or o_frag in t.object_value) + for t in context.triples + ) + assert found, ( + f"No triple with predicate={pred!r} linking subject containing " + f"{s_frag!r} to object containing {o_frag!r} in {context.triples}" + ) + + +@then( + r'the triples should not include predicate "(?P[^"]+)"' + r' with object_uri "(?P[^"]+)"' +) +def step_then_no_triple_pred_obj_uri(context: Context, pred: str, obj: str) -> None: + found = any(t.predicate == pred and t.object_uri == obj for t in context.triples) + assert not found, ( + f"Unexpected triple with predicate={pred!r} object_uri={obj!r} " + f"found in {context.triples}" + ) + + +@then( + r'the triples should not include predicate "(?P[^"]+)"' + r' with object_value "(?P[^"]+)"' +) +def step_then_no_triple_pred_obj_val(context: Context, pred: str, val: str) -> None: + found = any(t.predicate == pred and t.object_value == val for t in context.triples) + assert not found, ( + f"Unexpected triple with predicate={pred!r} object_value={val!r} " + f"found in {context.triples}" + ) + + +@then(r'the triples should include predicate "(?P[^"]+)"') +def step_then_triple_has_predicate(context: Context, pred: str) -> None: + found = any(t.predicate == pred for t in context.triples) + assert found, f"No triple with predicate={pred!r} in {context.triples}" + + +# Switch back to parse matcher for remaining steps. +use_step_matcher("parse") + + +# --------------------------------------------------------------------------- +# Then steps — error handling +# --------------------------------------------------------------------------- + + +@then("analyzing empty content with this analyzer should raise ValueError") +def step_then_empty_content_raises(context: Context) -> None: + try: + context.analyzer.analyze("", _SAMPLE_URI) + except ValueError: + return # Expected + raise AssertionError( + f"{type(context.analyzer).__name__}.analyze('', ...) did not raise ValueError" + ) + + +@then("analyzing whitespace-only content with this analyzer should raise ValueError") +def step_then_whitespace_only_raises(context: Context) -> None: + try: + context.analyzer.analyze(" \n\t ", _SAMPLE_URI) + except ValueError: + return # Expected + raise AssertionError( + f"{type(context.analyzer).__name__}.analyze(' \\n\\t ', ...) " + "did not raise ValueError" + ) + + +@then("analyzing content with empty resource_uri should raise ValueError") +def step_then_empty_resource_uri_raises(context: Context) -> None: + # Use a minimal valid content for each analyzer domain. + _domain_sample: dict[str, str] = { + "python": "class X:\n pass\n", + "markdown": "# Title\n", + "postgresql": "CREATE TABLE t (id SERIAL);\n", + "docker-compose": "services:\n web:\n image: nginx\n", + } + sample = _domain_sample.get(context.analyzer.domain, "content") + try: + context.analyzer.analyze(sample, "") + except ValueError: + return # Expected + raise AssertionError( + f"{type(context.analyzer).__name__}.analyze(..., '') did not raise ValueError" + ) + + +# --------------------------------------------------------------------------- +# Then steps — URI scheme and confidence +# --------------------------------------------------------------------------- + + +@then('every triple subject_uri should start with "{prefix}"') +def step_then_all_subject_uris_start_with(context: Context, prefix: str) -> None: + for triple in context.triples: + assert triple.subject_uri.startswith(prefix), ( + f"Triple subject_uri {triple.subject_uri!r} does not start with {prefix!r}" + ) + + +@then("every triple confidence should be {score:g}") +def step_then_all_confidence_scores(context: Context, score: float) -> None: + for triple in context.triples: + assert triple.confidence == score, ( + f"Triple {triple!r} has confidence={triple.confidence}, expected {score}" + ) diff --git a/robot/domain_analyzers.robot b/robot/domain_analyzers.robot new file mode 100644 index 000000000..24c4d7101 --- /dev/null +++ b/robot/domain_analyzers.robot @@ -0,0 +1,50 @@ +*** Settings *** +Documentation Integration smoke tests for domain-specific analyzers +Library Process +Suite Setup Log Domain Analyzers Robot Tests + +*** Variables *** +${HELPER} ${CURDIR}/helper_domain_analyzers.py + +*** Test Cases *** +Analyze Python Source + [Documentation] PythonAnalyzer extracts Module, Class, and Function triples + ${result}= Run Process python3 ${HELPER} analyze-python + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} analyze-python-ok + +Analyze Markdown Document + [Documentation] MarkdownAnalyzer extracts Document and Section triples + ${result}= Run Process python3 ${HELPER} analyze-markdown + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} analyze-markdown-ok + +Analyze PostgreSQL DDL + [Documentation] PostgreSQLAnalyzer extracts Table and Column triples + ${result}= Run Process python3 ${HELPER} analyze-postgresql + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} analyze-postgresql-ok + +Analyze Docker Compose YAML + [Documentation] DockerComposeAnalyzer extracts Service and DeploymentUnit triples + ${result}= Run Process python3 ${HELPER} analyze-docker-compose + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} analyze-docker-compose-ok + +Verify Analyzer Protocol Conformance + [Documentation] All four analyzers satisfy AnalyzerProtocol + ${result}= Run Process python3 ${HELPER} protocol-check + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} protocol-check-ok + +Verify Analyzer Registry Lookup + [Documentation] AnalyzerRegistry registers all four analyzers and resolves by extension + ${result}= Run Process python3 ${HELPER} registry-check + Log ${result.stdout} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} registry-check-ok diff --git a/robot/helper_domain_analyzers.py b/robot/helper_domain_analyzers.py new file mode 100644 index 000000000..b4dc3b881 --- /dev/null +++ b/robot/helper_domain_analyzers.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""Robot Framework helper for domain analyzer integration tests.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + +# Ensure worktree src is *first* on sys.path so it shadows any installed copy. +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC in sys.path: + sys.path.remove(_SRC) +sys.path.insert(0, _SRC) +# Clear cached cleveragents modules so our worktree src is used. +for _key in list(sys.modules): + if _key.startswith("cleveragents"): + del sys.modules[_key] + +from cleveragents.domain.models.acms.analyzers import ( # noqa: E402 + AnalyzerProtocol, + AnalyzerRegistry, +) +from cleveragents.domain.models.acms.docker_compose_analyzer import ( # noqa: E402 + DockerComposeAnalyzer, +) +from cleveragents.domain.models.acms.markdown_analyzer import ( # noqa: E402 + MarkdownAnalyzer, +) +from cleveragents.domain.models.acms.postgresql_analyzer import ( # noqa: E402 + PostgreSQLAnalyzer, +) +from cleveragents.domain.models.acms.python_analyzer import ( # noqa: E402 + PythonAnalyzer, +) + +# --------------------------------------------------------------------------- +# Sample content +# --------------------------------------------------------------------------- + +_SAMPLE_PYTHON = '''\ +"""User authentication service for the application.""" + +from __future__ import annotations + +import hashlib +import logging +from dataclasses import dataclass, field +from typing import Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class Credentials: + """Holds user login credentials.""" + + username: str + password_hash: str + is_active: bool = True + roles: list[str] = field(default_factory=list) + + +class AuthenticationService: + """Service responsible for authenticating users against the store.""" + + def __init__(self, secret_key: str) -> None: + self._secret_key = secret_key + self._sessions: dict[str, Credentials] = {} + + def authenticate(self, username: str, password: str) -> Optional[str]: + """Authenticate a user and return a session token. + + Args: + username: The login name. + password: The plaintext password to verify. + + Returns: + A session token string on success, or None on failure. + """ + pw_hash = hashlib.sha256( + (password + self._secret_key).encode() + ).hexdigest() + logger.info("Authentication attempt for user=%s", username) + return pw_hash if username else None + + def revoke_session(self, token: str) -> bool: + """Revoke an active session by its token.""" + if token in self._sessions: + del self._sessions[token] + return True + return False + + +def create_default_service() -> AuthenticationService: + """Factory function to create a service with default settings.""" + return AuthenticationService(secret_key="default-dev-key") +''' + +_SAMPLE_MARKDOWN = """\ +# Architecture Overview + +This document describes the high-level architecture of our platform. + +## Core Components + +The system is composed of several key services: + +- **API Gateway** — routes external traffic +- **Auth Service** — handles user authentication +- **Data Pipeline** — processes incoming events + +For more details see [the design doc](https://wiki.example.com/design). + +### API Gateway + +The gateway is built on [Envoy](https://envoyproxy.io) and handles +TLS termination, rate limiting, and request routing. + +```yaml +listeners: + - address: 0.0.0.0 + port: 8443 + tls: true +``` + +### Auth Service + +Token-based authentication using JWT. See [RFC 7519](https://tools.ietf.org/html/rfc7519). + +```python +def verify_token(token: str) -> Claims: + return jwt.decode(token, key=SECRET) +``` + +## Deployment + +The platform runs on Kubernetes with Helm charts managed in the +`infra/` directory. Refer to [ops runbook](./runbook.md) for details. +""" + +_SAMPLE_POSTGRESQL = """\ +CREATE SCHEMA IF NOT EXISTS app; + +CREATE TABLE app.users ( + id SERIAL PRIMARY KEY, + username VARCHAR(120) NOT NULL, + email VARCHAR(255) NOT NULL, + password VARCHAR(255) NOT NULL, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +CREATE TABLE app.roles ( + id SERIAL PRIMARY KEY, + name VARCHAR(80) NOT NULL, + description TEXT +); + +CREATE TABLE app.user_roles ( + user_id INTEGER NOT NULL, + role_id INTEGER NOT NULL, + granted_at TIMESTAMP DEFAULT NOW(), + PRIMARY KEY (user_id, role_id), + FOREIGN KEY (user_id) REFERENCES app.users (id), + FOREIGN KEY (role_id) REFERENCES app.roles (id) +); + +CREATE OR REPLACE VIEW app.active_users AS +SELECT u.id, u.username, u.email +FROM app.users u +WHERE u.is_active = TRUE; +""" + +_SAMPLE_DOCKER_COMPOSE = """\ +services: + web: + image: myapp/web:latest + ports: + - "8080:80" + - "8443:443" + environment: + DATABASE_URL: postgres://db:5432/app + REDIS_URL: redis://cache:6379 + LOG_LEVEL: info + depends_on: + - db + - cache + volumes: + - ./config:/app/config:ro + + db: + image: postgres:16-alpine + ports: + - "5432:5432" + environment: + POSTGRES_DB: app + POSTGRES_USER: appuser + POSTGRES_PASSWORD: secret + volumes: + - pgdata:/var/lib/postgresql/data + + cache: + image: redis:7-alpine + ports: + - "6379:6379" + + worker: + image: myapp/worker:latest + environment: + DATABASE_URL: postgres://db:5432/app + REDIS_URL: redis://cache:6379 + depends_on: + - db + - cache +""" + +# --------------------------------------------------------------------------- +# Helper utilities +# --------------------------------------------------------------------------- + + +def _has_object_uri(triples: list, uri_fragment: str) -> bool: + """Return True if any triple has *uri_fragment* in its object_uri.""" + return any(uri_fragment in t.object_uri for t in triples) + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + + +def _cmd_analyze_python() -> int: + """Create a PythonAnalyzer, analyze sample Python, verify triple types.""" + analyzer = PythonAnalyzer() + triples = analyzer.analyze(_SAMPLE_PYTHON, "src/auth/service.py") + + if not triples: + print("analyze-python-fail: no triples produced") + return 1 + + has_module = _has_object_uri(triples, "uko-code:Module") + has_class = _has_object_uri(triples, "uko-py:Class") + has_function = _has_object_uri(triples, "uko-py:Function") + + if not has_module: + print("analyze-python-fail: missing Module type") + return 1 + if not has_class: + print("analyze-python-fail: missing Class type") + return 1 + if not has_function: + print("analyze-python-fail: missing Function type") + return 1 + + print(f"analyze-python-ok: {len(triples)} triples (Module+Class+Function found)") + return 0 + + +def _cmd_analyze_markdown() -> int: + """Create a MarkdownAnalyzer, analyze sample markdown, verify triple types.""" + analyzer = MarkdownAnalyzer() + triples = analyzer.analyze(_SAMPLE_MARKDOWN, "docs/architecture.md") + + if not triples: + print("analyze-markdown-fail: no triples produced") + return 1 + + has_document = _has_object_uri(triples, "uko-doc:Document") + has_section = _has_object_uri(triples, "uko-doc:Section") + + if not has_document: + print("analyze-markdown-fail: missing Document type") + return 1 + if not has_section: + print("analyze-markdown-fail: missing Section type") + return 1 + + print(f"analyze-markdown-ok: {len(triples)} triples (Document+Section found)") + return 0 + + +def _cmd_analyze_postgresql() -> int: + """Create a PostgreSQLAnalyzer, analyze sample DDL, verify triple types.""" + analyzer = PostgreSQLAnalyzer() + triples = analyzer.analyze(_SAMPLE_POSTGRESQL, "db/migrations/001_init.sql") + + if not triples: + print("analyze-postgresql-fail: no triples produced") + return 1 + + has_table = _has_object_uri(triples, "uko-data:Table") + has_column = _has_object_uri(triples, "uko-data:Column") + + if not has_table: + print("analyze-postgresql-fail: missing Table type") + return 1 + if not has_column: + print("analyze-postgresql-fail: missing Column type") + return 1 + + print(f"analyze-postgresql-ok: {len(triples)} triples (Table+Column found)") + return 0 + + +def _cmd_analyze_docker_compose() -> int: + """Analyze sample compose YAML, verify triple types.""" + analyzer = DockerComposeAnalyzer() + triples = analyzer.analyze(_SAMPLE_DOCKER_COMPOSE, "infra/docker-compose.yml") + + if not triples: + print("analyze-docker-compose-fail: no triples produced") + return 1 + + has_service = _has_object_uri(triples, "uko-infra:Service") + has_deployment = _has_object_uri(triples, "uko-infra:DeploymentUnit") + + if not has_service: + print("analyze-docker-compose-fail: missing Service type") + return 1 + if not has_deployment: + print("analyze-docker-compose-fail: missing DeploymentUnit type") + return 1 + + print( + f"analyze-docker-compose-ok: {len(triples)} triples" + " (Service+DeploymentUnit found)" + ) + return 0 + + +def _cmd_protocol_check() -> int: + """Verify all 4 analyzers satisfy AnalyzerProtocol.""" + analyzer_classes = [ + PythonAnalyzer, + MarkdownAnalyzer, + PostgreSQLAnalyzer, + DockerComposeAnalyzer, + ] + + for cls in analyzer_classes: + inst = cls() + if not isinstance(inst, AnalyzerProtocol): + print( + f"protocol-check-fail: {cls.__name__} doesn't satisfy AnalyzerProtocol" + ) + return 1 + if not inst.domain: + print(f"protocol-check-fail: {cls.__name__} has empty domain") + return 1 + if not inst.supported_extensions: + print(f"protocol-check-fail: {cls.__name__} has no supported extensions") + return 1 + print( + f"protocol-check: {cls.__name__} ok" + f" (domain={inst.domain}, extensions={sorted(inst.supported_extensions)})" + ) + + print("protocol-check-ok") + return 0 + + +def _cmd_registry_check() -> int: + """Create AnalyzerRegistry, register all 4 analyzers, verify lookup by extension.""" + registry = AnalyzerRegistry() + analyzers = [ + PythonAnalyzer(), + MarkdownAnalyzer(), + PostgreSQLAnalyzer(), + DockerComposeAnalyzer(), + ] + + for analyzer in analyzers: + registry.register(analyzer) + + if len(registry) != 4: + print(f"registry-check-fail: expected 4 analyzers, got {len(registry)}") + return 1 + + # Verify extension lookup for each analyzer + extension_checks = { + ".py": "python", + ".pyi": "python", + ".md": "markdown", + ".markdown": "markdown", + ".sql": "postgresql", + ".ddl": "postgresql", + ".yml": "docker-compose", + ".yaml": "docker-compose", + } + + for ext, expected_domain in extension_checks.items(): + found = registry.get_for_extension(ext) + if found is None: + print(f"registry-check-fail: no analyzer found for '{ext}'") + return 1 + if found.domain != expected_domain: + print( + f"registry-check-fail: expected domain '{expected_domain}'" + f" for '{ext}', got '{found.domain}'" + ) + return 1 + print(f"registry-check: '{ext}' -> {found.domain}") + + # Verify unknown extension returns None + if registry.get_for_extension(".unknown") is not None: + print("registry-check-fail: expected None for unknown extension") + return 1 + + print(f"registry-check-ok: {len(registry)} analyzers, all extensions verified") + return 0 + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +_COMMANDS: dict[str, Callable[[], int]] = { + "analyze-python": _cmd_analyze_python, + "analyze-markdown": _cmd_analyze_markdown, + "analyze-postgresql": _cmd_analyze_postgresql, + "analyze-docker-compose": _cmd_analyze_docker_compose, + "protocol-check": _cmd_protocol_check, + "registry-check": _cmd_registry_check, +} + + +def main() -> int: + """Run the specified command.""" + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + print(f"Usage: {sys.argv[0]} <{'|'.join(_COMMANDS)}>") + return 2 + return _COMMANDS[sys.argv[1]]() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cleveragents/domain/models/acms/__init__.py b/src/cleveragents/domain/models/acms/__init__.py index 969d28f9f..5e87d6134 100644 --- a/src/cleveragents/domain/models/acms/__init__.py +++ b/src/cleveragents/domain/models/acms/__init__.py @@ -62,6 +62,8 @@ Analyzer types (from :mod:`~cleveragents.domain.models.acms.analyzers`): Concrete analyzers: - ``PythonAnalyzer`` -- AST-based Python source analyzer - ``MarkdownAnalyzer`` -- Heading/code-block/link Markdown analyzer +- ``PostgreSQLAnalyzer`` -- Regex-based DDL/SQL analyzer +- ``DockerComposeAnalyzer`` -- YAML-based Docker Compose analyzer Based on ``docs/specification.md`` ACMS / CRP sections and ADR-014. """ @@ -89,7 +91,11 @@ from cleveragents.domain.models.acms.crp import ( DetailLevelMap, FragmentProvenance, ) +from cleveragents.domain.models.acms.docker_compose_analyzer import ( + DockerComposeAnalyzer, +) from cleveragents.domain.models.acms.markdown_analyzer import MarkdownAnalyzer +from cleveragents.domain.models.acms.postgresql_analyzer import PostgreSQLAnalyzer from cleveragents.domain.models.acms.python_analyzer import PythonAnalyzer from cleveragents.domain.models.acms.scope_resolution import ( ResourceAliasResolver, @@ -151,6 +157,7 @@ __all__: list[str] = [ "ContextStrategyResult", "ContextTier", "DetailLevelMap", + "DockerComposeAnalyzer", "FragmentProvenance", "GraphBackend", "GraphResult", @@ -160,6 +167,7 @@ __all__: list[str] = [ "MarkdownAnalyzer", "PlanContext", "PlanDecisionContextStrategy", + "PostgreSQLAnalyzer", "PythonAnalyzer", "ResourceAliasResolver", "ResourceScope", diff --git a/src/cleveragents/domain/models/acms/_postgresql_helpers.py b/src/cleveragents/domain/models/acms/_postgresql_helpers.py new file mode 100644 index 000000000..f664f4139 --- /dev/null +++ b/src/cleveragents/domain/models/acms/_postgresql_helpers.py @@ -0,0 +1,440 @@ +"""PostgreSQL DDL parsing helpers for ``PostgreSQLAnalyzer``. + +Internal module containing compiled regex patterns, URI builders, +SQL comment stripping, and table-body parsing functions used by +:class:`~.postgresql_analyzer.PostgreSQLAnalyzer`. + +Not part of the public API -- do not import directly. +""" + +from __future__ import annotations + +import re + +from cleveragents.domain.models.acms.analyzers import UKOTriple + +__all__: list[str] = [] + +# --------------------------------------------------------------------------- +# Regex patterns +# --------------------------------------------------------------------------- + +CREATE_TABLE_RE = re.compile( + r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" + r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(", + re.IGNORECASE, +) + +CREATE_VIEW_RE = re.compile( + r"CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+" + r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s+AS\b", + re.IGNORECASE, +) + +CREATE_SCHEMA_RE = re.compile( + r"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?", + re.IGNORECASE, +) + +COLUMN_DEF_RE = re.compile( + r"^\s*\"?(\w+)\"?\s+" + r"((?:CHARACTER\s+VARYING|DOUBLE\s+PRECISION" + r"|TIME(?:STAMP)?\s+WITH(?:OUT)?\s+TIME\s+ZONE" + r"|\w+)" + r"(?:\s*\([^)]*\))?(?:\s*\[\s*\])?)", + re.IGNORECASE, +) + +NOT_NULL_RE = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) + +PRIMARY_KEY_INLINE_RE = re.compile(r"\bPRIMARY\s+KEY\b", re.IGNORECASE) + +FOREIGN_KEY_RE = re.compile( + r"FOREIGN\s+KEY\s*\(\s*([^)]+)\)\s*" + r"REFERENCES\s+(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(\s*([^)]+)\)", + re.IGNORECASE, +) + +TABLE_CONSTRAINT_PK_RE = re.compile( + r"PRIMARY\s+KEY\s*\(\s*([^)]+)\)", + re.IGNORECASE, +) + +# Keywords that start a line inside CREATE TABLE but are NOT column defs. +NON_COLUMN_KEYWORDS: frozenset[str] = frozenset( + { + "constraint", + "primary", + "foreign", + "unique", + "check", + "exclude", + "like", + "inherits", + } +) + +# --------------------------------------------------------------------------- +# URI helpers +# --------------------------------------------------------------------------- + +_SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.]") + + +def safe_name(name: str) -> str: + """Sanitise a SQL identifier for use in a URI path segment.""" + result = _SAFE_NAME_RE.sub("_", name)[:120] + return result if result else "_unknown_" + + +def table_uri(resource_uri: str, table_name: str, schema_name: str = "") -> str: + """Build a UKO table URI from a resource URI and table name.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return f"uko://data/table/{safe_name(resource_uri)}/{prefix}{safe_name(table_name)}" + + +def column_uri( + resource_uri: str, + table_name: str, + column_name: str, + schema_name: str = "", +) -> str: + """Build a UKO column URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return ( + f"uko://data/column/{safe_name(resource_uri)}" + f"/{prefix}{safe_name(table_name)}/{safe_name(column_name)}" + ) + + +def fk_uri( + resource_uri: str, + table_name: str, + column_name: str, + schema_name: str = "", +) -> str: + """Build a UKO foreign-key URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return ( + f"uko://data/fk/{safe_name(resource_uri)}" + f"/{prefix}{safe_name(table_name)}/{safe_name(column_name)}" + ) + + +def view_uri(resource_uri: str, view_name: str, schema_name: str = "") -> str: + """Build a UKO view URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return f"uko://data/view/{safe_name(resource_uri)}/{prefix}{safe_name(view_name)}" + + +def schema_uri(resource_uri: str, schema_name: str) -> str: + """Build a UKO schema URI.""" + return f"uko://data/schema/{safe_name(resource_uri)}/{safe_name(schema_name)}" + + +# --------------------------------------------------------------------------- +# SQL comment stripping +# --------------------------------------------------------------------------- + +_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) +_LINE_COMMENT_RE = re.compile(r"--[^\n]*") + + +def strip_sql_comments(content: str) -> str: + """Remove SQL block (``/* */``) and line (``--``) comments.""" + content = _BLOCK_COMMENT_RE.sub("", content) + return _LINE_COMMENT_RE.sub("", content) + + +# --------------------------------------------------------------------------- +# Parenthesis / body extraction +# --------------------------------------------------------------------------- + + +def extract_body(content: str, paren_start: int) -> str: + """Return text between balanced parentheses starting at *paren_start*. + + Respects single-quoted SQL string literals so that parentheses + inside strings (e.g. ``DEFAULT 'func(x)'``) do not affect the + depth counter. PostgreSQL ``''`` escape is handled by skipping + two consecutive single-quote characters. + """ + if paren_start >= len(content) or content[paren_start] != "(": + return "" + depth = 0 + in_string = False + i = paren_start + while i < len(content): + ch = content[i] + if in_string: + if ch == "'": + # '' is an escaped quote inside a string literal. + if i + 1 < len(content) and content[i + 1] == "'": + i += 2 + continue + in_string = False + else: + if ch == "'": + in_string = True + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return content[paren_start + 1 : i] + i += 1 + return "" + + +def split_entries(body: str) -> list[str]: + """Split a table body into entries by commas at depth 0. + + Respects single-quoted SQL string literals so that commas and + parentheses inside strings are ignored. + """ + entries: list[str] = [] + depth = 0 + in_string = False + current: list[str] = [] + i = 0 + while i < len(body): + ch = body[i] + if in_string: + current.append(ch) + if ch == "'": + # '' escape inside a string literal. + if i + 1 < len(body) and body[i + 1] == "'": + current.append(body[i + 1]) + i += 2 + continue + in_string = False + else: + if ch == "'": + in_string = True + current.append(ch) + elif ch == "(": + depth += 1 + current.append(ch) + elif ch == ")": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + entries.append("".join(current)) + current = [] + else: + current.append(ch) + i += 1 + tail = "".join(current).strip() + if tail: + entries.append(tail) + return entries + + +# --------------------------------------------------------------------------- +# Column emission +# --------------------------------------------------------------------------- + + +def emit_column_triples( + resource_uri: str, + table_name: str, + table_uri_val: str, + col_name: str, + col_type: str, + *, + schema_name: str = "", + is_nullable: bool, + is_primary_key: bool, +) -> list[UKOTriple]: + """Produce triples for a single column definition.""" + triples: list[UKOTriple] = [] + c_uri = column_uri(resource_uri, table_name, col_name, schema_name) + + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="rdf:type", + object_uri="uko-data:Column", + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="rdfs:label", + object_value=col_name, + ) + ) + triples.append( + UKOTriple( + subject_uri=table_uri_val, + predicate="uko:contains", + object_uri=c_uri, + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:dataType", + object_value=col_type, + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:isNullable", + object_value=str(is_nullable).lower(), + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:isPrimaryKey", + object_value=str(is_primary_key).lower(), + ) + ) + return triples + + +# --------------------------------------------------------------------------- +# Foreign key extraction +# --------------------------------------------------------------------------- + + +def extract_fk_triples( + fk_match: re.Match[str], + resource_uri: str, + table_name: str, + table_uri_val: str, + schema_name: str = "", +) -> list[UKOTriple]: + """Extract triples for a ``FOREIGN KEY`` constraint. + + Per spec section 42214-42217, ``uko-data:foreignKeyTo`` links + Column -> Column (domain = uko-data:Column, range = uko-data:Column). + + Supports multi-column foreign keys: each source/target column pair + gets its own ``foreignKeyTo`` edge. + """ + triples: list[UKOTriple] = [] + + source_cols = [ + c.strip().strip('"') for c in fk_match.group(1).split(",") if c.strip() + ] + ref_schema = fk_match.group(2) or schema_name + ref_table = fk_match.group(3) + target_cols = [ + c.strip().strip('"') for c in fk_match.group(4).split(",") if c.strip() + ] + + first_col = source_cols[0] if source_cols else "_unknown_" + fk_uri_val = fk_uri(resource_uri, table_name, first_col, schema_name) + + triples.append( + UKOTriple( + subject_uri=fk_uri_val, + predicate="rdf:type", + object_uri="uko-data:ForeignKey", + ) + ) + triples.append( + UKOTriple( + subject_uri=table_uri_val, + predicate="uko:contains", + object_uri=fk_uri_val, + ) + ) + + for src_col, tgt_col in zip(source_cols, target_cols, strict=False): + source_col_uri = column_uri(resource_uri, table_name, src_col, schema_name) + target_col_uri = column_uri(resource_uri, ref_table, tgt_col, ref_schema) + triples.append( + UKOTriple( + subject_uri=source_col_uri, + predicate="uko-data:foreignKeyTo", + object_uri=target_col_uri, + ) + ) + + return triples + + +# --------------------------------------------------------------------------- +# Table body parsing +# --------------------------------------------------------------------------- + + +def parse_table_body( + body: str, + resource_uri: str, + table_name: str, + table_uri_val: str, + schema_name: str = "", +) -> list[UKOTriple]: + """Parse column definitions and constraints inside a table body.""" + triples: list[UKOTriple] = [] + pk_columns: set[str] = set() + + # First pass: find table-level PRIMARY KEY constraints. + for pk_match in TABLE_CONSTRAINT_PK_RE.finditer(body): + cols = pk_match.group(1) + for col_name in cols.split(","): + cleaned = col_name.strip().strip('"').lower() + if cleaned: + pk_columns.add(cleaned) + + entries = split_entries(body) + + for entry in entries: + stripped = entry.strip() + if not stripped: + continue + + # Foreign key constraint + fk_match = FOREIGN_KEY_RE.search(stripped) + if fk_match: + triples.extend( + extract_fk_triples( + fk_match, + resource_uri, + table_name, + table_uri_val, + schema_name, + ) + ) + continue + + # Skip non-column constraint lines. + # F6 fix: quoted identifiers (e.g. "primary") are never keywords. + words = stripped.split() + raw_first = words[0] if words else "" + if ( + not raw_first.startswith('"') + and raw_first.strip('"').lower() in NON_COLUMN_KEYWORDS + ): + continue + + # Column definition + col_match = COLUMN_DEF_RE.match(stripped) + if col_match: + col_name = col_match.group(1) + col_type = col_match.group(2).strip() + + remainder = stripped[col_match.end() :] + is_not_null = bool(NOT_NULL_RE.search(remainder)) + is_pk_inline = bool(PRIMARY_KEY_INLINE_RE.search(remainder)) + is_pk = is_pk_inline or col_name.lower() in pk_columns + + triples.extend( + emit_column_triples( + resource_uri, + table_name, + table_uri_val, + col_name, + col_type, + schema_name=schema_name, + is_nullable=not (is_not_null or is_pk), + is_primary_key=is_pk, + ) + ) + + return triples diff --git a/src/cleveragents/domain/models/acms/docker_compose_analyzer.py b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py new file mode 100644 index 000000000..9dda92e40 --- /dev/null +++ b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py @@ -0,0 +1,451 @@ +"""DockerComposeAnalyzer — YAML-based UKO triple extraction from Compose files. + +Parses Docker Compose YAML content and extracts: + +- Deployment unit declaration (``uko-infra:DeploymentUnit``). +- Service definitions (``uko-infra:Service``). +- Port mappings with exposure relationships (``uko-infra:Port``). +- Environment variable declarations (``uko-infra:EnvironmentVariable``). +- Service dependency edges (``uko-infra:connectsTo``). +- Volume mount mappings (``uko:contains``). + +All extracted elements are represented as ``UKOTriple`` instances with +``uko://`` URI schemes following the UKO ontology hierarchy: + +- Layer 0 core: ``uko:contains`` +- Layer 2 infra: ``uko-infra:Service``, ``uko-infra:Port``, + ``uko-infra:EnvironmentVariable``, ``uko-infra:DeploymentUnit``, + ``uko-infra:exposes``, ``uko-infra:connectsTo`` + +Since ``.yml``/``.yaml`` extensions are shared with other YAML formats, +the analyzer validates that the document contains a ``services`` key +before extracting triples. + +Based on ``docs/specification.md`` §42285-42331 — DockerComposeAnalyzer. +""" + +from __future__ import annotations + +import logging +import re + +import yaml + +from cleveragents.domain.models.acms.analyzers import AnalyzerProtocol, UKOTriple + +__all__ = ["DockerComposeAnalyzer"] + +logger = logging.getLogger(__name__) + +# Maximum content size in bytes before parsing. Guards against +# billion-laughs-style alias expansion attacks in YAML. +_MAX_COMPOSE_BYTES = 1_048_576 # 1 MiB + +# --------------------------------------------------------------------------- +# URI helpers +# --------------------------------------------------------------------------- + +_SAFE_RE = re.compile(r"[^a-zA-Z0-9_.-]") + + +def _safe(text: str) -> str: + """Sanitise text for use in a URI path segment. + + Truncates to 120 characters. Callers should be aware that very + long inputs sharing a common prefix may collide after truncation. + """ + result = _SAFE_RE.sub("_", text).strip("_")[:120] + return result if result else "_unknown_" + + +def _deployment_uri(resource_uri: str) -> str: + """Build a UKO deployment-unit URI from a resource URI.""" + return f"uko://infra/deployment/{_safe(resource_uri)}" + + +def _service_uri(resource_uri: str, service_name: str) -> str: + """Build a UKO service URI.""" + return f"uko://infra/service/{_safe(resource_uri)}/{_safe(service_name)}" + + +def _port_uri(resource_uri: str, service_name: str, port_mapping: str) -> str: + """Build a UKO port URI.""" + return ( + f"uko://infra/port/{_safe(resource_uri)}" + f"/{_safe(service_name)}/{_safe(port_mapping)}" + ) + + +def _env_uri(resource_uri: str, service_name: str, var_name: str) -> str: + """Build a UKO environment-variable URI.""" + return ( + f"uko://infra/env/{_safe(resource_uri)}/{_safe(service_name)}/{_safe(var_name)}" + ) + + +def _volume_uri(resource_uri: str, service_name: str, volume_mapping: str) -> str: + """Build a UKO volume URI.""" + return ( + f"uko://infra/volume/{_safe(resource_uri)}" + f"/{_safe(service_name)}/{_safe(volume_mapping)}" + ) + + +# --------------------------------------------------------------------------- +# DockerComposeAnalyzer +# --------------------------------------------------------------------------- + + +class DockerComposeAnalyzer: + """YAML-based Docker Compose analyzer producing UKO triples. + + Satisfies :class:`AnalyzerProtocol`. Handles ``.yml`` and ``.yaml`` + files that contain Docker Compose definitions. Validates the + presence of a ``services`` key before extracting triples. Returns + an empty list for unparsable or non-Compose YAML. + + Example:: + + analyzer = DockerComposeAnalyzer() + triples = analyzer.analyze( + "services:\\n web:\\n image: nginx\\n", + "infra/docker-compose.yml", + ) + """ + + @property + def supported_extensions(self) -> frozenset[str]: + """File extensions handled by this analyzer.""" + return frozenset({".yml", ".yaml"}) + + @property + def domain(self) -> str: + """Human-readable domain label.""" + return "docker-compose" + + def analyze(self, content: str, resource_uri: str) -> list[UKOTriple]: + """Parse *content* as Docker Compose YAML and extract UKO triples. + + Args: + content: Raw YAML text. + resource_uri: Canonical URI of the resource. + + Returns: + List of ``UKOTriple`` instances. Returns an empty list when + the content is unparsable or not a Docker Compose document. + + Raises: + ValueError: If *content* or *resource_uri* is empty. + """ + if not content or not content.strip(): + raise ValueError("content must not be empty.") + if not resource_uri or not resource_uri.strip(): + raise ValueError("resource_uri must not be empty.") + + # Size guard: reject oversized inputs to mitigate billion-laughs + # alias-expansion attacks (quadratic memory via nested YAML + # anchors/aliases). 1 MiB is generous for any Compose file. + if len(content) > _MAX_COMPOSE_BYTES: + logger.warning( + "DockerComposeAnalyzer: content exceeds %d byte limit; skipping '%s'", + _MAX_COMPOSE_BYTES, + resource_uri, + ) + return [] + + try: + data = yaml.safe_load(content) + except yaml.YAMLError: + logger.warning( + "DockerComposeAnalyzer: YAML parse error in '%s'; returning empty", + resource_uri, + exc_info=True, + ) + return [] + + # safe_load may return None for empty documents, or a non-dict scalar. + if not isinstance(data, dict): + return [] + + # Docker Compose detection: require 'services' key. Older Compose + # files (v2/v3) always have a 'services' mapping alongside 'version'. + if "services" not in data: + return [] + + triples: list[UKOTriple] = [] + deploy_uri = _deployment_uri(resource_uri) + + # Deployment unit declaration + triples.append( + UKOTriple( + subject_uri=deploy_uri, + predicate="rdf:type", + object_uri="uko-infra:DeploymentUnit", + ) + ) + triples.append( + UKOTriple( + subject_uri=deploy_uri, + predicate="rdfs:label", + object_value=resource_uri, + ) + ) + + services = data.get("services") + if not isinstance(services, dict): + # Has 'version' but no valid 'services' — return just the + # deployment unit triples. + return triples + + for service_name, service_def in services.items(): + svc_uri = _service_uri(resource_uri, str(service_name)) + + # Service declaration + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="rdf:type", + object_uri="uko-infra:Service", + ) + ) + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="rdfs:label", + object_value=str(service_name), + ) + ) + + # Containment: deployment unit -> service + triples.append( + UKOTriple( + subject_uri=deploy_uri, + predicate="uko:contains", + object_uri=svc_uri, + ) + ) + + # Guard against null service definitions (e.g. `web:` with + # no body). + if not isinstance(service_def, dict): + continue + + # Ports + triples.extend( + self._extract_ports(service_def, resource_uri, service_name, svc_uri) + ) + + # Environment variables + triples.extend( + self._extract_environment( + service_def, resource_uri, service_name, svc_uri + ) + ) + + # Dependencies + triples.extend(self._extract_depends_on(service_def, resource_uri, svc_uri)) + + # Volumes + triples.extend( + self._extract_volumes(service_def, resource_uri, service_name, svc_uri) + ) + + return triples + + # -- Internal extraction helpers ------------------------------------------ + + def _extract_ports( + self, + service_def: dict, + resource_uri: str, + service_name: str, + svc_uri: str, + ) -> list[UKOTriple]: + """Extract port mapping triples from a service definition.""" + triples: list[UKOTriple] = [] + ports = service_def.get("ports") + if not isinstance(ports, list): + return triples + + for port_entry in ports: + # Normalise dict-form ports (extended syntax) to a + # human-readable string instead of Python dict repr. + if isinstance(port_entry, dict): + target = port_entry.get("target", "") + published = port_entry.get("published", "") + port_str = f"{published}:{target}" if published else str(target) + else: + port_str = str(port_entry) + p_uri = _port_uri(resource_uri, service_name, port_str) + + triples.append( + UKOTriple( + subject_uri=p_uri, + predicate="rdf:type", + object_uri="uko-infra:Port", + ) + ) + # NOTE: Spec §42327-42330 defines exposes with range + # uko-infra:Endpoint, but Docker Compose port bindings map + # most naturally to uko-infra:Port. We use exposes here for + # its semantic meaning ("service exposes this port") rather + # than creating an intermediate Endpoint wrapper. + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="uko-infra:exposes", + object_uri=p_uri, + ) + ) + triples.append( + UKOTriple( + subject_uri=p_uri, + predicate="rdfs:label", + object_value=port_str, + ) + ) + + return triples + + def _extract_environment( + self, + service_def: dict, + resource_uri: str, + service_name: str, + svc_uri: str, + ) -> list[UKOTriple]: + """Extract environment variable triples from a service definition.""" + triples: list[UKOTriple] = [] + env = service_def.get("environment") + if env is None: + return triples + + var_names: list[str] = [] + + if isinstance(env, dict): + # Mapping form: environment: { KEY: value, ... } + var_names = [str(k) for k in env] + elif isinstance(env, list): + # List form: environment: [ "KEY=value", ... ] + for entry in env: + entry_str = str(entry) + name = entry_str.split("=", 1)[0] + var_names.append(name) + + for var_name in var_names: + e_uri = _env_uri(resource_uri, service_name, var_name) + + triples.append( + UKOTriple( + subject_uri=e_uri, + predicate="rdf:type", + object_uri="uko-infra:EnvironmentVariable", + ) + ) + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="uko:contains", + object_uri=e_uri, + ) + ) + triples.append( + UKOTriple( + subject_uri=e_uri, + predicate="rdfs:label", + object_value=var_name, + ) + ) + + return triples + + def _extract_depends_on( + self, + service_def: dict, + resource_uri: str, + svc_uri: str, + ) -> list[UKOTriple]: + """Extract service dependency triples from ``depends_on``.""" + triples: list[UKOTriple] = [] + depends_on = service_def.get("depends_on") + if depends_on is None: + return triples + + dep_names: list[str] = [] + + if isinstance(depends_on, str): + # Single string form: depends_on: db + dep_names = [depends_on] + elif isinstance(depends_on, list): + # Short form: depends_on: [db, redis] + dep_names = [str(d) for d in depends_on] + elif isinstance(depends_on, dict): + # Long form: depends_on: { db: { condition: ... }, ... } + dep_names = [str(k) for k in depends_on] + + for dep_name in dep_names: + dep_uri = _service_uri(resource_uri, dep_name) + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="uko-infra:connectsTo", + object_uri=dep_uri, + ) + ) + + return triples + + def _extract_volumes( + self, + service_def: dict, + resource_uri: str, + service_name: str, + svc_uri: str, + ) -> list[UKOTriple]: + """Extract volume mapping triples from a service definition.""" + triples: list[UKOTriple] = [] + volumes = service_def.get("volumes") + if not isinstance(volumes, list): + return triples + + for vol_entry in volumes: + # Normalise dict-form volumes (long syntax) to a + # human-readable "source:target" string. + if isinstance(vol_entry, dict): + source = vol_entry.get("source", "") + target = vol_entry.get("target", "") + vol_str = f"{source}:{target}" if source else str(target) + else: + vol_str = str(vol_entry) + v_uri = _volume_uri(resource_uri, service_name, vol_str) + + # Volume mounts are typed as ConfigKey per the uko-infra + # vocabulary (spec §42303-42305) — the nearest fit since + # no dedicated uko-infra:Volume class exists in the spec. + triples.append( + UKOTriple( + subject_uri=v_uri, + predicate="rdf:type", + object_uri="uko-infra:ConfigKey", + ) + ) + triples.append( + UKOTriple( + subject_uri=svc_uri, + predicate="uko:contains", + object_uri=v_uri, + ) + ) + triples.append( + UKOTriple( + subject_uri=v_uri, + predicate="rdfs:label", + object_value=vol_str, + ) + ) + + return triples + + +# Protocol compliance assertion (zero-cost at import time). +_: type[AnalyzerProtocol] = DockerComposeAnalyzer diff --git a/src/cleveragents/domain/models/acms/postgresql_analyzer.py b/src/cleveragents/domain/models/acms/postgresql_analyzer.py new file mode 100644 index 000000000..9f9f6b2b7 --- /dev/null +++ b/src/cleveragents/domain/models/acms/postgresql_analyzer.py @@ -0,0 +1,310 @@ +"""PostgreSQLAnalyzer -- regex-based extraction of UKO triples from DDL. + +Parses PostgreSQL DDL content and extracts: + +- Table declarations (``uko-data:Table``). +- Column definitions with type, nullability, and primary key + (``uko-data:Column``). +- Foreign key constraints (``uko-data:ForeignKey``). +- View declarations (``uko-data:View``). +- Schema declarations and qualified-name detection + (``uko-data:Schema``). + +All extracted elements are represented as ``UKOTriple`` instances with +``uko://`` URI schemes following the UKO ontology hierarchy: + +- Layer 0 core: ``uko:contains`` +- Layer 1 data: ``uko-data:Table``, ``uko-data:Column``, + ``uko-data:View``, ``uko-data:Schema``, ``uko-data:ForeignKey`` + +Uses line-by-line regex parsing; no external SQL library required. + +Based on ``docs/specification.md`` section 42151-42272 ACMS Extensions -- +PostgreSQLAnalyzer. +""" + +from __future__ import annotations + +import logging + +from cleveragents.domain.models.acms._postgresql_helpers import ( + CREATE_SCHEMA_RE, + CREATE_TABLE_RE, + CREATE_VIEW_RE, + extract_body, + parse_table_body, + schema_uri, + strip_sql_comments, + table_uri, + view_uri, +) +from cleveragents.domain.models.acms.analyzers import AnalyzerProtocol, UKOTriple + +__all__ = ["PostgreSQLAnalyzer"] + +logger = logging.getLogger(__name__) + + +class PostgreSQLAnalyzer: + """Regex-based PostgreSQL DDL analyzer producing UKO triples. + + Satisfies :class:`AnalyzerProtocol`. Handles ``.sql`` and ``.ddl`` + files. Gracefully returns an empty list for unparsable content. + + .. note:: + + The ``uko-data:Database`` container is intentionally omitted + because DDL files do not inherently carry database context. + Provenance is added by ``UKOIndexer`` (section 43205), not by + analyzers. + + Example:: + + analyzer = PostgreSQLAnalyzer() + triples = analyzer.analyze( + "CREATE TABLE users (id SERIAL PRIMARY KEY);\\n", + "db/migrations/001.sql", + ) + """ + + @property + def supported_extensions(self) -> frozenset[str]: + """File extensions handled by this analyzer.""" + return frozenset({".sql", ".ddl"}) + + @property + def domain(self) -> str: + """Human-readable domain label.""" + return "postgresql" + + def analyze(self, content: str, resource_uri: str) -> list[UKOTriple]: + """Parse *content* as PostgreSQL DDL and extract UKO triples. + + Args: + content: Raw DDL/SQL text. + resource_uri: Canonical URI of the resource. + + Returns: + List of ``UKOTriple`` instances. Returns an empty list when + no DDL statements are detected. + + Raises: + ValueError: If *content* or *resource_uri* is empty or + whitespace-only. + """ + if not content or not content.strip(): + raise ValueError("content must not be empty.") + if not resource_uri or not resource_uri.strip(): + raise ValueError("resource_uri must not be empty.") + + content = strip_sql_comments(content) + + triples: list[UKOTriple] = [] + schemas_seen: set[str] = set() + tables_seen: set[str] = set() + + try: + triples.extend(self._extract_schemas(content, resource_uri, schemas_seen)) + triples.extend( + self._extract_tables(content, resource_uri, schemas_seen, tables_seen) + ) + triples.extend(self._extract_views(content, resource_uri, schemas_seen)) + except Exception: + logger.warning( + "PostgreSQLAnalyzer: parse error in '%s'; returning partial results", + resource_uri, + exc_info=True, + ) + + return triples + + # -- Schema extraction ---------------------------------------------------- + + def _extract_schemas( + self, + content: str, + resource_uri: str, + schemas_seen: set[str], + ) -> list[UKOTriple]: + """Extract triples for explicit ``CREATE SCHEMA`` statements.""" + triples: list[UKOTriple] = [] + + for match in CREATE_SCHEMA_RE.finditer(content): + s_name = match.group(1).lower() + if s_name in schemas_seen: + continue + schemas_seen.add(s_name) + s_uri = schema_uri(resource_uri, s_name) + + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="rdf:type", + object_uri="uko-data:Schema", + ) + ) + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="rdfs:label", + object_value=s_name, + ) + ) + + return triples + + def _ensure_schema( + self, + schema_name: str, + resource_uri: str, + schemas_seen: set[str], + triples: list[UKOTriple], + ) -> str: + """Ensure a schema node exists and return its URI.""" + schema_lower = schema_name.lower() + s_uri = schema_uri(resource_uri, schema_lower) + + if schema_lower not in schemas_seen: + schemas_seen.add(schema_lower) + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="rdf:type", + object_uri="uko-data:Schema", + ) + ) + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="rdfs:label", + object_value=schema_lower, + ) + ) + + return s_uri + + # -- Table extraction ----------------------------------------------------- + + def _extract_tables( + self, + content: str, + resource_uri: str, + schemas_seen: set[str], + tables_seen: set[str] | None = None, + ) -> list[UKOTriple]: + """Extract triples for ``CREATE TABLE`` statements.""" + triples: list[UKOTriple] = [] + if tables_seen is None: + tables_seen = set() + + for match in CREATE_TABLE_RE.finditer(content): + s_name = match.group(1) or "" + t_name = match.group(2) + schema_lower = s_name.lower() if s_name else "" + t_uri = table_uri(resource_uri, t_name, schema_lower) + + table_key = f"{schema_lower}.{t_name.lower()}" + if table_key in tables_seen: + continue + tables_seen.add(table_key) + + triples.append( + UKOTriple( + subject_uri=t_uri, + predicate="rdf:type", + object_uri="uko-data:Table", + ) + ) + triples.append( + UKOTriple( + subject_uri=t_uri, + predicate="rdfs:label", + object_value=t_name, + ) + ) + + if s_name: + s_uri = self._ensure_schema(s_name, resource_uri, schemas_seen, triples) + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="uko:contains", + object_uri=t_uri, + ) + ) + + body = extract_body(content, match.end() - 1) + if body: + triples.extend( + parse_table_body(body, resource_uri, t_name, t_uri, schema_lower) + ) + + return triples + + # -- View extraction ------------------------------------------------------ + + def _extract_views( + self, + content: str, + resource_uri: str, + schemas_seen: set[str], + ) -> list[UKOTriple]: + """Extract triples for ``CREATE VIEW`` statements. + + Emits ``uko-data:viewDefinition`` with the SQL body of the view + (spec section 42263-42266). + """ + triples: list[UKOTriple] = [] + + for match in CREATE_VIEW_RE.finditer(content): + s_name = match.group(1) or "" + schema_lower = s_name.lower() if s_name else "" + v_name = match.group(2) + v_uri = view_uri(resource_uri, v_name, schema_lower) + + triples.append( + UKOTriple( + subject_uri=v_uri, + predicate="rdf:type", + object_uri="uko-data:View", + ) + ) + triples.append( + UKOTriple( + subject_uri=v_uri, + predicate="rdfs:label", + object_value=v_name, + ) + ) + + as_start = match.end() + semi_pos = content.find(";", as_start) + view_sql = ( + content[as_start:semi_pos].strip() + if semi_pos != -1 + else content[as_start:].strip() + ) + if view_sql: + triples.append( + UKOTriple( + subject_uri=v_uri, + predicate="uko-data:viewDefinition", + object_value=view_sql, + ) + ) + + if s_name: + s_uri = self._ensure_schema(s_name, resource_uri, schemas_seen, triples) + triples.append( + UKOTriple( + subject_uri=s_uri, + predicate="uko:contains", + object_uri=v_uri, + ) + ) + + return triples + + +# Protocol compliance assertion (zero-cost at import time). +_: type[AnalyzerProtocol] = PostgreSQLAnalyzer diff --git a/vulture_whitelist.py b/vulture_whitelist.py index 0eab344aa..24b35be74 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -829,3 +829,7 @@ serve # noqa: B018, F821 # cleveragents.lsp.server.LspServer.facade — lazy ACP facade property facade # noqa: B018, F821 + +# Domain-specific analyzers — public API (issue #588) +PostgreSQLAnalyzer # noqa: B018, F821 +DockerComposeAnalyzer # noqa: B018, F821