From 13618fb8d58d270e8166db9e0efa7e98e0db347f Mon Sep 17 00:00:00 2001 From: Hamza Khyari Date: Fri, 6 Mar 2026 20:07:03 +0000 Subject: [PATCH] fix(acms): address PR #611 review findings F1-F7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - F1/F2 [P1]: Split postgresql_analyzer.py (695→310 lines) by extracting regex patterns, URI builders, and parsing functions into _postgresql_helpers.py (440 lines). Split domain_analyzers.feature (605→217 lines) into separate postgresql_analyzer.feature (201) and docker_compose_analyzer.feature (199). - F3 [P1]: Fix _extract_body and _split_entries to track single-quoted SQL string literals so parentheses inside DEFAULT/CHECK values (e.g. 'func(x)') do not corrupt depth counting. Added regression scenarios. - F4 [P2]: Remove prohibited `# type: ignore[arg-type]` by adding a static protocol assertion for _DuplicatePyAnalyzer. - F5 [P2]: Add 1 MiB size guard before yaml.safe_load in DockerComposeAnalyzer to mitigate billion-laughs alias expansion attacks. - F6 [P2]: Fix quoted-keyword column names (e.g. "primary") being silently dropped by checking raw_first.startswith('"') before keyword filtering. Added regression scenario. - F7 [P2]: Out-of-scope file changes resolved by rebase onto master (changes already merged via other PRs). All nox gates pass: lint (0 errors), typecheck (0 errors), security_scan, behave (57 scenarios, 211 steps, 0 failures). --- features/docker_compose_analyzer.feature | 199 ++++++++ features/domain_analyzers.feature | 388 -------------- features/postgresql_analyzer.feature | 201 ++++++++ features/steps/domain_analyzer_steps.py | 4 +- .../domain/models/acms/_postgresql_helpers.py | 440 ++++++++++++++++ .../models/acms/docker_compose_analyzer.py | 15 + .../domain/models/acms/postgresql_analyzer.py | 483 ++---------------- 7 files changed, 907 insertions(+), 823 deletions(-) create mode 100644 features/docker_compose_analyzer.feature create mode 100644 features/postgresql_analyzer.feature create mode 100644 src/cleveragents/domain/models/acms/_postgresql_helpers.py diff --git a/features/docker_compose_analyzer.feature b/features/docker_compose_analyzer.feature new file mode 100644 index 000000000..bbbe36174 --- /dev/null +++ b/features/docker_compose_analyzer.feature @@ -0,0 +1,199 @@ +@phase2 @acms @analyzer @docker_compose_analyzer +Feature: DockerComposeAnalyzer + As a CleverAgents developer + I want a Docker Compose analyzer that produces UKO triples from YAML files + So that the ACMS can index infrastructure definitions into the UKO knowledge graph + + Scenario: Analyze services + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + api: + image: node:18 + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:DeploymentUnit" + And the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" + And the triples should include predicate "uko:contains" linking subject "deployment" to object "web" + + Scenario: Analyze ports + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + ports: + - "8080:80" + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" + And the triples should include predicate "uko-infra:exposes" + + Scenario: Analyze environment variables + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + api: + image: node:18 + environment: + DATABASE_URL: postgres://localhost/db + NODE_ENV: production + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" + And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" + And the triples should include predicate "rdfs:label" with object_value "NODE_ENV" + + Scenario: Analyze depends_on + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: + - api + - redis + api: + image: node:18 + redis: + image: redis:7 + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze depends_on as single string + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: db + db: + image: postgres + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze depends_on as dict form + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + depends_on: + db: + condition: service_healthy + db: + image: postgres + """ + Then the triples should include predicate "uko-infra:connectsTo" + + Scenario: Analyze volumes + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + volumes: + - ./data:/app/data + """ + Then the triples should include predicate "uko:contains" linking subject "web" to object "volume" + + Scenario: Analyze environment as list form + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + api: + image: node:18 + environment: + - DATABASE_URL=postgres://localhost/db + - NODE_ENV=production + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" + And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" + + Scenario: Analyze service with null body + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + placeholder: + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" + And the triples should include predicate "rdfs:label" with object_value "placeholder" + + Scenario: Analyze dict-form port mapping + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + ports: + - target: 80 + published: 8080 + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" + And the triples should include predicate "rdfs:label" with object_value "8080:80" + + Scenario: Analyze dict-form volume mapping + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + image: nginx + volumes: + - type: bind + source: ./data + target: /data + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-infra:ConfigKey" + And the triples should include predicate "rdfs:label" with object_value "./data:/data" + + Scenario: Analyze empty Docker Compose content raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + Scenario: Analyze whitespace-only Docker Compose content raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing whitespace-only content with this analyzer should raise ValueError + + Scenario: Analyze Docker Compose with empty resource_uri raises ValueError + Given a DockerComposeAnalyzer analyzer + Then analyzing content with empty resource_uri should raise ValueError + + Scenario: Analyze version-only YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + version: "3.8" + """ + Then no triples should be returned + + Scenario: Analyze non-compose YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + name: my-application + settings: + debug: true + log_level: info + """ + Then no triples should be returned + + Scenario: Analyze invalid YAML returns empty list + Given a DockerComposeAnalyzer analyzer + When I analyze the Docker Compose content: + """ + services: + web: + ports: + - this: is: broken: yaml: [ + """ + Then no triples should be returned diff --git a/features/domain_analyzers.feature b/features/domain_analyzers.feature index eae51ebaa..4dfd82120 100644 --- a/features/domain_analyzers.feature +++ b/features/domain_analyzers.feature @@ -193,394 +193,6 @@ Feature: Domain-Specific Analyzers Then the triples should include predicate "rdf:type" with object_uri "uko-doc:Document" And the triples should not include predicate "rdf:type" with object_uri "uko-doc:Section" - # --------------------------------------------------------------------------- - # PostgreSQLAnalyzer - # --------------------------------------------------------------------------- - - @postgresql_analyzer - Scenario: Analyze CREATE TABLE - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE users ( - id SERIAL, - name VARCHAR(100) - ); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" - And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" - And the triples should include predicate "uko-data:dataType" - - @postgresql_analyzer - Scenario: Analyze table with NOT NULL and PRIMARY KEY - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE accounts ( - id SERIAL PRIMARY KEY, - email VARCHAR(255) NOT NULL - ); - """ - Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" - And the triples should include predicate "uko-data:isNullable" with object_value "false" - - @postgresql_analyzer - Scenario: Analyze FOREIGN KEY - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE orders ( - id SERIAL PRIMARY KEY, - user_id INTEGER NOT NULL, - FOREIGN KEY (user_id) REFERENCES users(id) - ); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" - And the triples should include predicate "uko-data:foreignKeyTo" - - @postgresql_analyzer - Scenario: Analyze CREATE VIEW - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE VIEW active_users AS - SELECT * FROM users WHERE active = true; - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" - And the triples should include predicate "rdfs:label" with object_value "active_users" - - @postgresql_analyzer - Scenario: Analyze CREATE SCHEMA - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE SCHEMA IF NOT EXISTS app; - CREATE TABLE app.users ( - id SERIAL PRIMARY KEY - ); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:Schema" - And the triples should include predicate "rdfs:label" with object_value "app" - And the triples should include predicate "uko:contains" linking subject "schema" to object "users" - - @postgresql_analyzer - Scenario: Analyze schema-qualified table includes schema in column URIs - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE myschema.items ( - id SERIAL PRIMARY KEY, - name VARCHAR(100) - ); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" - And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" - And the triples should include predicate "uko:contains" linking subject "myschema" to object "items" - - @postgresql_analyzer - Scenario: Analyze table with composite PRIMARY KEY - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE user_roles ( - user_id INTEGER NOT NULL, - role_id INTEGER NOT NULL, - PRIMARY KEY (user_id, role_id) - ); - """ - Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" - - @postgresql_analyzer - Scenario: Analyze empty SQL content raises ValueError - Given a PostgreSQLAnalyzer analyzer - Then analyzing empty content with this analyzer should raise ValueError - - @postgresql_analyzer - Scenario: Analyze whitespace-only SQL content raises ValueError - Given a PostgreSQLAnalyzer analyzer - Then analyzing whitespace-only content with this analyzer should raise ValueError - - @postgresql_analyzer - Scenario: Analyze SQL with empty resource_uri raises ValueError - Given a PostgreSQLAnalyzer analyzer - Then analyzing content with empty resource_uri should raise ValueError - - @postgresql_analyzer - Scenario: Analyze SQL with comments does not produce phantom triples - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - -- CREATE TABLE phantom (id INT); - /* CREATE VIEW ghost AS SELECT 1; */ - CREATE TABLE real_table (id SERIAL PRIMARY KEY); - """ - Then the triples should include predicate "rdfs:label" with object_value "real_table" - And the triples should not include predicate "rdfs:label" with object_value "phantom" - And the triples should not include predicate "rdfs:label" with object_value "ghost" - - @postgresql_analyzer - Scenario: Analyze multi-column FOREIGN KEY - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE user_roles ( - user_id INTEGER NOT NULL, - role_id INTEGER NOT NULL, - PRIMARY KEY (user_id, role_id), - FOREIGN KEY (user_id, role_id) REFERENCES lookup(uid, rid) - ); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" - And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "uid" - And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "rid" - - @postgresql_analyzer - Scenario: Analyze CREATE VIEW captures viewDefinition - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE VIEW active_users AS - SELECT id, name FROM users WHERE active = true; - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" - And the triples should include predicate "uko-data:viewDefinition" - - @postgresql_analyzer - Scenario: Duplicate CREATE TABLE is deduplicated - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - CREATE TABLE items (id SERIAL); - CREATE TABLE items (id SERIAL, name TEXT); - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" - And the number of Table type triples should be 1 - - @postgresql_analyzer - Scenario: Analyze invalid SQL returns empty list - Given a PostgreSQLAnalyzer analyzer - When I analyze the SQL content: - """ - THIS IS NOT VALID SQL AT ALL; - JUST RANDOM TEXT HERE; - """ - Then no triples should be returned - - # --------------------------------------------------------------------------- - # DockerComposeAnalyzer - # --------------------------------------------------------------------------- - - @docker_compose_analyzer - Scenario: Analyze services - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - api: - image: node:18 - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:DeploymentUnit" - And the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" - And the triples should include predicate "uko:contains" linking subject "deployment" to object "web" - - @docker_compose_analyzer - Scenario: Analyze ports - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - ports: - - "8080:80" - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" - And the triples should include predicate "uko-infra:exposes" - - @docker_compose_analyzer - Scenario: Analyze environment variables - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - api: - image: node:18 - environment: - DATABASE_URL: postgres://localhost/db - NODE_ENV: production - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" - And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" - And the triples should include predicate "rdfs:label" with object_value "NODE_ENV" - - @docker_compose_analyzer - Scenario: Analyze depends_on - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - depends_on: - - api - - redis - api: - image: node:18 - redis: - image: redis:7 - """ - Then the triples should include predicate "uko-infra:connectsTo" - - @docker_compose_analyzer - Scenario: Analyze depends_on as single string - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - depends_on: db - db: - image: postgres - """ - Then the triples should include predicate "uko-infra:connectsTo" - - @docker_compose_analyzer - Scenario: Analyze depends_on as dict form - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - depends_on: - db: - condition: service_healthy - db: - image: postgres - """ - Then the triples should include predicate "uko-infra:connectsTo" - - @docker_compose_analyzer - Scenario: Analyze volumes - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - volumes: - - ./data:/app/data - """ - Then the triples should include predicate "uko:contains" linking subject "web" to object "volume" - - @docker_compose_analyzer - Scenario: Analyze environment as list form - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - api: - image: node:18 - environment: - - DATABASE_URL=postgres://localhost/db - - NODE_ENV=production - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:EnvironmentVariable" - And the triples should include predicate "rdfs:label" with object_value "DATABASE_URL" - - @docker_compose_analyzer - Scenario: Analyze service with null body - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - placeholder: - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Service" - And the triples should include predicate "rdfs:label" with object_value "placeholder" - - @docker_compose_analyzer - Scenario: Analyze dict-form port mapping - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - ports: - - target: 80 - published: 8080 - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:Port" - And the triples should include predicate "rdfs:label" with object_value "8080:80" - - @docker_compose_analyzer - Scenario: Analyze dict-form volume mapping - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - image: nginx - volumes: - - type: bind - source: ./data - target: /data - """ - Then the triples should include predicate "rdf:type" with object_uri "uko-infra:ConfigKey" - And the triples should include predicate "rdfs:label" with object_value "./data:/data" - - @docker_compose_analyzer - Scenario: Analyze empty Docker Compose content raises ValueError - Given a DockerComposeAnalyzer analyzer - Then analyzing empty content with this analyzer should raise ValueError - - @docker_compose_analyzer - Scenario: Analyze whitespace-only Docker Compose content raises ValueError - Given a DockerComposeAnalyzer analyzer - Then analyzing whitespace-only content with this analyzer should raise ValueError - - @docker_compose_analyzer - Scenario: Analyze Docker Compose with empty resource_uri raises ValueError - Given a DockerComposeAnalyzer analyzer - Then analyzing content with empty resource_uri should raise ValueError - - @docker_compose_analyzer - Scenario: Analyze version-only YAML returns empty list - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - version: "3.8" - """ - Then no triples should be returned - - @docker_compose_analyzer - Scenario: Analyze non-compose YAML returns empty list - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - name: my-application - settings: - debug: true - log_level: info - """ - Then no triples should be returned - - @docker_compose_analyzer - Scenario: Analyze invalid YAML returns empty list - Given a DockerComposeAnalyzer analyzer - When I analyze the Docker Compose content: - """ - services: - web: - ports: - - this: is: broken: yaml: [ - """ - Then no triples should be returned - # --------------------------------------------------------------------------- # Cross-analyzer # --------------------------------------------------------------------------- diff --git a/features/postgresql_analyzer.feature b/features/postgresql_analyzer.feature new file mode 100644 index 000000000..f733e0416 --- /dev/null +++ b/features/postgresql_analyzer.feature @@ -0,0 +1,201 @@ +@phase2 @acms @analyzer @postgresql_analyzer +Feature: PostgreSQLAnalyzer + As a CleverAgents developer + I want a PostgreSQL DDL analyzer that produces UKO triples from SQL files + So that the ACMS can index database schemas into the UKO knowledge graph + + Scenario: Analyze CREATE TABLE + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE users ( + id SERIAL, + name VARCHAR(100) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" + And the triples should include predicate "uko-data:dataType" + + Scenario: Analyze table with NOT NULL and PRIMARY KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE accounts ( + id SERIAL PRIMARY KEY, + email VARCHAR(255) NOT NULL + ); + """ + Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" + And the triples should include predicate "uko-data:isNullable" with object_value "false" + + Scenario: Analyze FOREIGN KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" + And the triples should include predicate "uko-data:foreignKeyTo" + + Scenario: Analyze CREATE VIEW + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE VIEW active_users AS + SELECT * FROM users WHERE active = true; + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" + And the triples should include predicate "rdfs:label" with object_value "active_users" + + Scenario: Analyze CREATE SCHEMA + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE SCHEMA IF NOT EXISTS app; + CREATE TABLE app.users ( + id SERIAL PRIMARY KEY + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Schema" + And the triples should include predicate "rdfs:label" with object_value "app" + And the triples should include predicate "uko:contains" linking subject "schema" to object "users" + + Scenario: Analyze schema-qualified table includes schema in column URIs + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE myschema.items ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the triples should include predicate "rdf:type" with object_uri "uko-data:Column" + And the triples should include predicate "uko:contains" linking subject "myschema" to object "items" + + Scenario: Analyze table with composite PRIMARY KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE user_roles ( + user_id INTEGER NOT NULL, + role_id INTEGER NOT NULL, + PRIMARY KEY (user_id, role_id) + ); + """ + Then the triples should include predicate "uko-data:isPrimaryKey" with object_value "true" + + Scenario: Analyze empty SQL content raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing empty content with this analyzer should raise ValueError + + Scenario: Analyze whitespace-only SQL content raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing whitespace-only content with this analyzer should raise ValueError + + Scenario: Analyze SQL with empty resource_uri raises ValueError + Given a PostgreSQLAnalyzer analyzer + Then analyzing content with empty resource_uri should raise ValueError + + Scenario: Analyze SQL with comments does not produce phantom triples + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + -- CREATE TABLE phantom (id INT); + /* CREATE VIEW ghost AS SELECT 1; */ + CREATE TABLE real_table (id SERIAL PRIMARY KEY); + """ + Then the triples should include predicate "rdfs:label" with object_value "real_table" + And the triples should not include predicate "rdfs:label" with object_value "phantom" + And the triples should not include predicate "rdfs:label" with object_value "ghost" + + Scenario: Analyze multi-column FOREIGN KEY + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE user_roles ( + user_id INTEGER NOT NULL, + role_id INTEGER NOT NULL, + PRIMARY KEY (user_id, role_id), + FOREIGN KEY (user_id, role_id) REFERENCES lookup(uid, rid) + ); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:ForeignKey" + And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "uid" + And the triples should include predicate "uko-data:foreignKeyTo" with object_uri containing "rid" + + Scenario: Analyze CREATE VIEW captures viewDefinition + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE VIEW active_users AS + SELECT id, name FROM users WHERE active = true; + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:View" + And the triples should include predicate "uko-data:viewDefinition" + + Scenario: Duplicate CREATE TABLE is deduplicated + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE items (id SERIAL); + CREATE TABLE items (id SERIAL, name TEXT); + """ + Then the triples should include predicate "rdf:type" with object_uri "uko-data:Table" + And the number of Table type triples should be 1 + + Scenario: Analyze invalid SQL returns empty list + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + THIS IS NOT VALID SQL AT ALL; + JUST RANDOM TEXT HERE; + """ + Then no triples should be returned + + # -- Regression tests for F3 (string-literal parentheses) -- + + Scenario: DEFAULT with parentheses in string literal does not truncate columns + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE config ( + col1 TEXT DEFAULT 'hello(world)', + col2 INTEGER + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "col1" + And the triples should include predicate "rdfs:label" with object_value "col2" + + Scenario: CHECK constraint with string containing parentheses + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE validated ( + status TEXT CHECK (status IN ('active(1)', 'inactive(0)')), + amount INTEGER + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "status" + And the triples should include predicate "rdfs:label" with object_value "amount" + + # -- Regression test for F6 (quoted keyword column names) -- + + Scenario: Quoted keyword column names are not silently dropped + Given a PostgreSQLAnalyzer analyzer + When I analyze the SQL content: + """ + CREATE TABLE reserved_words ( + "primary" INTEGER, + "check" VARCHAR(50), + normal_col TEXT + ); + """ + Then the triples should include predicate "rdfs:label" with object_value "primary" + And the triples should include predicate "rdfs:label" with object_value "check" + And the triples should include predicate "rdfs:label" with object_value "normal_col" diff --git a/features/steps/domain_analyzer_steps.py b/features/steps/domain_analyzer_steps.py index d8e892519..ee4b3478f 100644 --- a/features/steps/domain_analyzer_steps.py +++ b/features/steps/domain_analyzer_steps.py @@ -121,7 +121,9 @@ def step_when_register_duplicate_py(context: Context) -> None: def analyze(self, content: str, resource_uri: str) -> list[UKOTriple]: return [] # pragma: no cover - context.registry.register(_DuplicatePyAnalyzer()) # type: ignore[arg-type] + _dup: type[AnalyzerProtocol] = _DuplicatePyAnalyzer + del _dup # only used for static type-checking assertion + context.registry.register(_DuplicatePyAnalyzer()) @when('I look up extension "{ext}" in the registry') diff --git a/src/cleveragents/domain/models/acms/_postgresql_helpers.py b/src/cleveragents/domain/models/acms/_postgresql_helpers.py new file mode 100644 index 000000000..f664f4139 --- /dev/null +++ b/src/cleveragents/domain/models/acms/_postgresql_helpers.py @@ -0,0 +1,440 @@ +"""PostgreSQL DDL parsing helpers for ``PostgreSQLAnalyzer``. + +Internal module containing compiled regex patterns, URI builders, +SQL comment stripping, and table-body parsing functions used by +:class:`~.postgresql_analyzer.PostgreSQLAnalyzer`. + +Not part of the public API -- do not import directly. +""" + +from __future__ import annotations + +import re + +from cleveragents.domain.models.acms.analyzers import UKOTriple + +__all__: list[str] = [] + +# --------------------------------------------------------------------------- +# Regex patterns +# --------------------------------------------------------------------------- + +CREATE_TABLE_RE = re.compile( + r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" + r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(", + re.IGNORECASE, +) + +CREATE_VIEW_RE = re.compile( + r"CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+" + r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s+AS\b", + re.IGNORECASE, +) + +CREATE_SCHEMA_RE = re.compile( + r"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?", + re.IGNORECASE, +) + +COLUMN_DEF_RE = re.compile( + r"^\s*\"?(\w+)\"?\s+" + r"((?:CHARACTER\s+VARYING|DOUBLE\s+PRECISION" + r"|TIME(?:STAMP)?\s+WITH(?:OUT)?\s+TIME\s+ZONE" + r"|\w+)" + r"(?:\s*\([^)]*\))?(?:\s*\[\s*\])?)", + re.IGNORECASE, +) + +NOT_NULL_RE = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) + +PRIMARY_KEY_INLINE_RE = re.compile(r"\bPRIMARY\s+KEY\b", re.IGNORECASE) + +FOREIGN_KEY_RE = re.compile( + r"FOREIGN\s+KEY\s*\(\s*([^)]+)\)\s*" + r"REFERENCES\s+(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(\s*([^)]+)\)", + re.IGNORECASE, +) + +TABLE_CONSTRAINT_PK_RE = re.compile( + r"PRIMARY\s+KEY\s*\(\s*([^)]+)\)", + re.IGNORECASE, +) + +# Keywords that start a line inside CREATE TABLE but are NOT column defs. +NON_COLUMN_KEYWORDS: frozenset[str] = frozenset( + { + "constraint", + "primary", + "foreign", + "unique", + "check", + "exclude", + "like", + "inherits", + } +) + +# --------------------------------------------------------------------------- +# URI helpers +# --------------------------------------------------------------------------- + +_SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.]") + + +def safe_name(name: str) -> str: + """Sanitise a SQL identifier for use in a URI path segment.""" + result = _SAFE_NAME_RE.sub("_", name)[:120] + return result if result else "_unknown_" + + +def table_uri(resource_uri: str, table_name: str, schema_name: str = "") -> str: + """Build a UKO table URI from a resource URI and table name.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return f"uko://data/table/{safe_name(resource_uri)}/{prefix}{safe_name(table_name)}" + + +def column_uri( + resource_uri: str, + table_name: str, + column_name: str, + schema_name: str = "", +) -> str: + """Build a UKO column URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return ( + f"uko://data/column/{safe_name(resource_uri)}" + f"/{prefix}{safe_name(table_name)}/{safe_name(column_name)}" + ) + + +def fk_uri( + resource_uri: str, + table_name: str, + column_name: str, + schema_name: str = "", +) -> str: + """Build a UKO foreign-key URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return ( + f"uko://data/fk/{safe_name(resource_uri)}" + f"/{prefix}{safe_name(table_name)}/{safe_name(column_name)}" + ) + + +def view_uri(resource_uri: str, view_name: str, schema_name: str = "") -> str: + """Build a UKO view URI.""" + prefix = f"{safe_name(schema_name)}." if schema_name else "" + return f"uko://data/view/{safe_name(resource_uri)}/{prefix}{safe_name(view_name)}" + + +def schema_uri(resource_uri: str, schema_name: str) -> str: + """Build a UKO schema URI.""" + return f"uko://data/schema/{safe_name(resource_uri)}/{safe_name(schema_name)}" + + +# --------------------------------------------------------------------------- +# SQL comment stripping +# --------------------------------------------------------------------------- + +_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) +_LINE_COMMENT_RE = re.compile(r"--[^\n]*") + + +def strip_sql_comments(content: str) -> str: + """Remove SQL block (``/* */``) and line (``--``) comments.""" + content = _BLOCK_COMMENT_RE.sub("", content) + return _LINE_COMMENT_RE.sub("", content) + + +# --------------------------------------------------------------------------- +# Parenthesis / body extraction +# --------------------------------------------------------------------------- + + +def extract_body(content: str, paren_start: int) -> str: + """Return text between balanced parentheses starting at *paren_start*. + + Respects single-quoted SQL string literals so that parentheses + inside strings (e.g. ``DEFAULT 'func(x)'``) do not affect the + depth counter. PostgreSQL ``''`` escape is handled by skipping + two consecutive single-quote characters. + """ + if paren_start >= len(content) or content[paren_start] != "(": + return "" + depth = 0 + in_string = False + i = paren_start + while i < len(content): + ch = content[i] + if in_string: + if ch == "'": + # '' is an escaped quote inside a string literal. + if i + 1 < len(content) and content[i + 1] == "'": + i += 2 + continue + in_string = False + else: + if ch == "'": + in_string = True + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return content[paren_start + 1 : i] + i += 1 + return "" + + +def split_entries(body: str) -> list[str]: + """Split a table body into entries by commas at depth 0. + + Respects single-quoted SQL string literals so that commas and + parentheses inside strings are ignored. + """ + entries: list[str] = [] + depth = 0 + in_string = False + current: list[str] = [] + i = 0 + while i < len(body): + ch = body[i] + if in_string: + current.append(ch) + if ch == "'": + # '' escape inside a string literal. + if i + 1 < len(body) and body[i + 1] == "'": + current.append(body[i + 1]) + i += 2 + continue + in_string = False + else: + if ch == "'": + in_string = True + current.append(ch) + elif ch == "(": + depth += 1 + current.append(ch) + elif ch == ")": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + entries.append("".join(current)) + current = [] + else: + current.append(ch) + i += 1 + tail = "".join(current).strip() + if tail: + entries.append(tail) + return entries + + +# --------------------------------------------------------------------------- +# Column emission +# --------------------------------------------------------------------------- + + +def emit_column_triples( + resource_uri: str, + table_name: str, + table_uri_val: str, + col_name: str, + col_type: str, + *, + schema_name: str = "", + is_nullable: bool, + is_primary_key: bool, +) -> list[UKOTriple]: + """Produce triples for a single column definition.""" + triples: list[UKOTriple] = [] + c_uri = column_uri(resource_uri, table_name, col_name, schema_name) + + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="rdf:type", + object_uri="uko-data:Column", + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="rdfs:label", + object_value=col_name, + ) + ) + triples.append( + UKOTriple( + subject_uri=table_uri_val, + predicate="uko:contains", + object_uri=c_uri, + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:dataType", + object_value=col_type, + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:isNullable", + object_value=str(is_nullable).lower(), + ) + ) + triples.append( + UKOTriple( + subject_uri=c_uri, + predicate="uko-data:isPrimaryKey", + object_value=str(is_primary_key).lower(), + ) + ) + return triples + + +# --------------------------------------------------------------------------- +# Foreign key extraction +# --------------------------------------------------------------------------- + + +def extract_fk_triples( + fk_match: re.Match[str], + resource_uri: str, + table_name: str, + table_uri_val: str, + schema_name: str = "", +) -> list[UKOTriple]: + """Extract triples for a ``FOREIGN KEY`` constraint. + + Per spec section 42214-42217, ``uko-data:foreignKeyTo`` links + Column -> Column (domain = uko-data:Column, range = uko-data:Column). + + Supports multi-column foreign keys: each source/target column pair + gets its own ``foreignKeyTo`` edge. + """ + triples: list[UKOTriple] = [] + + source_cols = [ + c.strip().strip('"') for c in fk_match.group(1).split(",") if c.strip() + ] + ref_schema = fk_match.group(2) or schema_name + ref_table = fk_match.group(3) + target_cols = [ + c.strip().strip('"') for c in fk_match.group(4).split(",") if c.strip() + ] + + first_col = source_cols[0] if source_cols else "_unknown_" + fk_uri_val = fk_uri(resource_uri, table_name, first_col, schema_name) + + triples.append( + UKOTriple( + subject_uri=fk_uri_val, + predicate="rdf:type", + object_uri="uko-data:ForeignKey", + ) + ) + triples.append( + UKOTriple( + subject_uri=table_uri_val, + predicate="uko:contains", + object_uri=fk_uri_val, + ) + ) + + for src_col, tgt_col in zip(source_cols, target_cols, strict=False): + source_col_uri = column_uri(resource_uri, table_name, src_col, schema_name) + target_col_uri = column_uri(resource_uri, ref_table, tgt_col, ref_schema) + triples.append( + UKOTriple( + subject_uri=source_col_uri, + predicate="uko-data:foreignKeyTo", + object_uri=target_col_uri, + ) + ) + + return triples + + +# --------------------------------------------------------------------------- +# Table body parsing +# --------------------------------------------------------------------------- + + +def parse_table_body( + body: str, + resource_uri: str, + table_name: str, + table_uri_val: str, + schema_name: str = "", +) -> list[UKOTriple]: + """Parse column definitions and constraints inside a table body.""" + triples: list[UKOTriple] = [] + pk_columns: set[str] = set() + + # First pass: find table-level PRIMARY KEY constraints. + for pk_match in TABLE_CONSTRAINT_PK_RE.finditer(body): + cols = pk_match.group(1) + for col_name in cols.split(","): + cleaned = col_name.strip().strip('"').lower() + if cleaned: + pk_columns.add(cleaned) + + entries = split_entries(body) + + for entry in entries: + stripped = entry.strip() + if not stripped: + continue + + # Foreign key constraint + fk_match = FOREIGN_KEY_RE.search(stripped) + if fk_match: + triples.extend( + extract_fk_triples( + fk_match, + resource_uri, + table_name, + table_uri_val, + schema_name, + ) + ) + continue + + # Skip non-column constraint lines. + # F6 fix: quoted identifiers (e.g. "primary") are never keywords. + words = stripped.split() + raw_first = words[0] if words else "" + if ( + not raw_first.startswith('"') + and raw_first.strip('"').lower() in NON_COLUMN_KEYWORDS + ): + continue + + # Column definition + col_match = COLUMN_DEF_RE.match(stripped) + if col_match: + col_name = col_match.group(1) + col_type = col_match.group(2).strip() + + remainder = stripped[col_match.end() :] + is_not_null = bool(NOT_NULL_RE.search(remainder)) + is_pk_inline = bool(PRIMARY_KEY_INLINE_RE.search(remainder)) + is_pk = is_pk_inline or col_name.lower() in pk_columns + + triples.extend( + emit_column_triples( + resource_uri, + table_name, + table_uri_val, + col_name, + col_type, + schema_name=schema_name, + is_nullable=not (is_not_null or is_pk), + is_primary_key=is_pk, + ) + ) + + return triples diff --git a/src/cleveragents/domain/models/acms/docker_compose_analyzer.py b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py index 1473a56fb..9dda92e40 100644 --- a/src/cleveragents/domain/models/acms/docker_compose_analyzer.py +++ b/src/cleveragents/domain/models/acms/docker_compose_analyzer.py @@ -37,6 +37,10 @@ __all__ = ["DockerComposeAnalyzer"] logger = logging.getLogger(__name__) +# Maximum content size in bytes before parsing. Guards against +# billion-laughs-style alias expansion attacks in YAML. +_MAX_COMPOSE_BYTES = 1_048_576 # 1 MiB + # --------------------------------------------------------------------------- # URI helpers # --------------------------------------------------------------------------- @@ -138,6 +142,17 @@ class DockerComposeAnalyzer: if not resource_uri or not resource_uri.strip(): raise ValueError("resource_uri must not be empty.") + # Size guard: reject oversized inputs to mitigate billion-laughs + # alias-expansion attacks (quadratic memory via nested YAML + # anchors/aliases). 1 MiB is generous for any Compose file. + if len(content) > _MAX_COMPOSE_BYTES: + logger.warning( + "DockerComposeAnalyzer: content exceeds %d byte limit; skipping '%s'", + _MAX_COMPOSE_BYTES, + resource_uri, + ) + return [] + try: data = yaml.safe_load(content) except yaml.YAMLError: diff --git a/src/cleveragents/domain/models/acms/postgresql_analyzer.py b/src/cleveragents/domain/models/acms/postgresql_analyzer.py index 2ec5dfd84..9f9f6b2b7 100644 --- a/src/cleveragents/domain/models/acms/postgresql_analyzer.py +++ b/src/cleveragents/domain/models/acms/postgresql_analyzer.py @@ -1,4 +1,4 @@ -"""PostgreSQLAnalyzer — regex-based extraction of UKO triples from DDL files. +"""PostgreSQLAnalyzer -- regex-based extraction of UKO triples from DDL. Parses PostgreSQL DDL content and extracts: @@ -19,154 +19,31 @@ All extracted elements are represented as ``UKOTriple`` instances with Uses line-by-line regex parsing; no external SQL library required. -Based on ``docs/specification.md`` §42151-42272 ACMS Extensions — +Based on ``docs/specification.md`` section 42151-42272 ACMS Extensions -- PostgreSQLAnalyzer. """ from __future__ import annotations import logging -import re +from cleveragents.domain.models.acms._postgresql_helpers import ( + CREATE_SCHEMA_RE, + CREATE_TABLE_RE, + CREATE_VIEW_RE, + extract_body, + parse_table_body, + schema_uri, + strip_sql_comments, + table_uri, + view_uri, +) from cleveragents.domain.models.acms.analyzers import AnalyzerProtocol, UKOTriple __all__ = ["PostgreSQLAnalyzer"] logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Regex patterns -# --------------------------------------------------------------------------- - -_CREATE_TABLE_RE = re.compile( - r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?" - r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(", - re.IGNORECASE, -) - -_CREATE_VIEW_RE = re.compile( - r"CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+" - r"(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s+AS\b", - re.IGNORECASE, -) - -_CREATE_SCHEMA_RE = re.compile( - r"CREATE\s+SCHEMA\s+(?:IF\s+NOT\s+EXISTS\s+)?\"?(\w+)\"?", - re.IGNORECASE, -) - -_COLUMN_DEF_RE = re.compile( - r"^\s*\"?(\w+)\"?\s+" - r"((?:CHARACTER\s+VARYING|DOUBLE\s+PRECISION|TIME(?:STAMP)?\s+WITH(?:OUT)?\s+TIME\s+ZONE" - r"|\w+)" - r"(?:\s*\([^)]*\))?(?:\s*\[\s*\])?)", - re.IGNORECASE, -) - -_NOT_NULL_RE = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE) - -_PRIMARY_KEY_INLINE_RE = re.compile(r"\bPRIMARY\s+KEY\b", re.IGNORECASE) - -_FOREIGN_KEY_RE = re.compile( - r"FOREIGN\s+KEY\s*\(\s*([^)]+)\)\s*" - r"REFERENCES\s+(?:\"?(\w+)\"?\.)?\"?(\w+)\"?\s*\(\s*([^)]+)\)", - re.IGNORECASE, -) - -_TABLE_CONSTRAINT_PK_RE = re.compile( - r"PRIMARY\s+KEY\s*\(\s*([^)]+)\)", - re.IGNORECASE, -) - -# Keywords that start a line inside CREATE TABLE but are NOT column defs. -_NON_COLUMN_KEYWORDS = frozenset( - { - "constraint", - "primary", - "foreign", - "unique", - "check", - "exclude", - "like", - "inherits", - } -) - -# --------------------------------------------------------------------------- -# URI helpers -# --------------------------------------------------------------------------- - -_SAFE_NAME_RE = re.compile(r"[^a-zA-Z0-9_.]") - - -def _safe_name(name: str) -> str: - """Sanitise a SQL identifier for use in a URI path segment.""" - result = _SAFE_NAME_RE.sub("_", name)[:120] - return result if result else "_unknown_" - - -def _table_uri(resource_uri: str, table_name: str, schema_name: str = "") -> str: - """Build a UKO table URI from a resource URI and table name.""" - prefix = f"{_safe_name(schema_name)}." if schema_name else "" - return ( - f"uko://data/table/{_safe_name(resource_uri)}/{prefix}{_safe_name(table_name)}" - ) - - -def _column_uri( - resource_uri: str, - table_name: str, - column_name: str, - schema_name: str = "", -) -> str: - """Build a UKO column URI.""" - prefix = f"{_safe_name(schema_name)}." if schema_name else "" - return ( - f"uko://data/column/{_safe_name(resource_uri)}" - f"/{prefix}{_safe_name(table_name)}/{_safe_name(column_name)}" - ) - - -def _fk_uri( - resource_uri: str, table_name: str, column_name: str, schema_name: str = "" -) -> str: - """Build a UKO foreign-key URI.""" - prefix = f"{_safe_name(schema_name)}." if schema_name else "" - return ( - f"uko://data/fk/{_safe_name(resource_uri)}" - f"/{prefix}{_safe_name(table_name)}/{_safe_name(column_name)}" - ) - - -def _view_uri(resource_uri: str, view_name: str, schema_name: str = "") -> str: - """Build a UKO view URI.""" - prefix = f"{_safe_name(schema_name)}." if schema_name else "" - return f"uko://data/view/{_safe_name(resource_uri)}/{prefix}{_safe_name(view_name)}" - - -def _schema_uri(resource_uri: str, schema_name: str) -> str: - """Build a UKO schema URI.""" - return f"uko://data/schema/{_safe_name(resource_uri)}/{_safe_name(schema_name)}" - - -# --------------------------------------------------------------------------- -# Comment stripping -# --------------------------------------------------------------------------- - -_BLOCK_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) -_LINE_COMMENT_RE = re.compile(r"--[^\n]*") - - -def _strip_sql_comments(content: str) -> str: - """Remove SQL block (``/* */``) and line (``--``) comments.""" - content = _BLOCK_COMMENT_RE.sub("", content) - return _LINE_COMMENT_RE.sub("", content) - - -# --------------------------------------------------------------------------- -# PostgreSQLAnalyzer -# --------------------------------------------------------------------------- - class PostgreSQLAnalyzer: """Regex-based PostgreSQL DDL analyzer producing UKO triples. @@ -178,7 +55,7 @@ class PostgreSQLAnalyzer: The ``uko-data:Database`` container is intentionally omitted because DDL files do not inherently carry database context. - Provenance is added by ``UKOIndexer`` (§43205), not by + Provenance is added by ``UKOIndexer`` (section 43205), not by analyzers. Example:: @@ -212,16 +89,15 @@ class PostgreSQLAnalyzer: no DDL statements are detected. Raises: - ValueError: If *content* or *resource_uri* is empty. + ValueError: If *content* or *resource_uri* is empty or + whitespace-only. """ if not content or not content.strip(): raise ValueError("content must not be empty.") if not resource_uri or not resource_uri.strip(): raise ValueError("resource_uri must not be empty.") - # Strip SQL comments before parsing to avoid phantom triples - # from commented-out DDL (common in migration files). - content = _strip_sql_comments(content) + content = strip_sql_comments(content) triples: list[UKOTriple] = [] schemas_seen: set[str] = set() @@ -242,7 +118,7 @@ class PostgreSQLAnalyzer: return triples - # -- Internal extraction helpers ------------------------------------------ + # -- Schema extraction ---------------------------------------------------- def _extract_schemas( self, @@ -253,12 +129,12 @@ class PostgreSQLAnalyzer: """Extract triples for explicit ``CREATE SCHEMA`` statements.""" triples: list[UKOTriple] = [] - for match in _CREATE_SCHEMA_RE.finditer(content): - schema_name = match.group(1).lower() - if schema_name in schemas_seen: + for match in CREATE_SCHEMA_RE.finditer(content): + s_name = match.group(1).lower() + if s_name in schemas_seen: continue - schemas_seen.add(schema_name) - s_uri = _schema_uri(resource_uri, schema_name) + schemas_seen.add(s_name) + s_uri = schema_uri(resource_uri, s_name) triples.append( UKOTriple( @@ -271,7 +147,7 @@ class PostgreSQLAnalyzer: UKOTriple( subject_uri=s_uri, predicate="rdfs:label", - object_value=schema_name, + object_value=s_name, ) ) @@ -286,7 +162,7 @@ class PostgreSQLAnalyzer: ) -> str: """Ensure a schema node exists and return its URI.""" schema_lower = schema_name.lower() - s_uri = _schema_uri(resource_uri, schema_lower) + s_uri = schema_uri(resource_uri, schema_lower) if schema_lower not in schemas_seen: schemas_seen.add(schema_lower) @@ -307,6 +183,8 @@ class PostgreSQLAnalyzer: return s_uri + # -- Table extraction ----------------------------------------------------- + def _extract_tables( self, content: str, @@ -314,24 +192,22 @@ class PostgreSQLAnalyzer: schemas_seen: set[str], tables_seen: set[str] | None = None, ) -> list[UKOTriple]: - """Extract triples for ``CREATE TABLE`` statements and their contents.""" + """Extract triples for ``CREATE TABLE`` statements.""" triples: list[UKOTriple] = [] if tables_seen is None: tables_seen = set() - for match in _CREATE_TABLE_RE.finditer(content): - schema_name = match.group(1) or "" - table_name = match.group(2) - schema_lower = schema_name.lower() if schema_name else "" - t_uri = _table_uri(resource_uri, table_name, schema_lower) + for match in CREATE_TABLE_RE.finditer(content): + s_name = match.group(1) or "" + t_name = match.group(2) + schema_lower = s_name.lower() if s_name else "" + t_uri = table_uri(resource_uri, t_name, schema_lower) - # H4: Skip duplicate CREATE TABLE for same qualified name. - table_key = f"{schema_lower}.{table_name.lower()}" + table_key = f"{schema_lower}.{t_name.lower()}" if table_key in tables_seen: continue tables_seen.add(table_key) - # Table type declaration triples.append( UKOTriple( subject_uri=t_uri, @@ -339,21 +215,16 @@ class PostgreSQLAnalyzer: object_uri="uko-data:Table", ) ) - - # Table label triples.append( UKOTriple( subject_uri=t_uri, predicate="rdfs:label", - object_value=table_name, + object_value=t_name, ) ) - # Schema containment - if schema_name: - s_uri = self._ensure_schema( - schema_name, resource_uri, schemas_seen, triples - ) + if s_name: + s_uri = self._ensure_schema(s_name, resource_uri, schemas_seen, triples) triples.append( UKOTriple( subject_uri=s_uri, @@ -362,263 +233,15 @@ class PostgreSQLAnalyzer: ) ) - # Extract the parenthesised body of the CREATE TABLE - body = self._extract_body(content, match.end() - 1) + body = extract_body(content, match.end() - 1) if body: triples.extend( - self._parse_table_body( - body, resource_uri, table_name, t_uri, schema_lower - ) + parse_table_body(body, resource_uri, t_name, t_uri, schema_lower) ) return triples - def _extract_body(self, content: str, paren_start: int) -> str: - """Return the text between balanced parentheses starting at *paren_start*.""" - if paren_start >= len(content) or content[paren_start] != "(": - return "" - depth = 0 - for i in range(paren_start, len(content)): - if content[i] == "(": - depth += 1 - elif content[i] == ")": - depth -= 1 - if depth == 0: - return content[paren_start + 1 : i] - return "" - - def _parse_table_body( - self, - body: str, - resource_uri: str, - table_name: str, - table_uri: str, - schema_name: str = "", - ) -> list[UKOTriple]: - """Parse the column definitions and constraints inside a table body.""" - triples: list[UKOTriple] = [] - pk_columns: set[str] = set() - - # First pass: find table-level PRIMARY KEY constraints to mark - # columns later. - for pk_match in _TABLE_CONSTRAINT_PK_RE.finditer(body): - cols = pk_match.group(1) - for col_name in cols.split(","): - cleaned = col_name.strip().strip('"').lower() - if cleaned: - pk_columns.add(cleaned) - - # Split into top-level comma-separated entries (respecting parens). - entries = self._split_entries(body) - - for entry in entries: - stripped = entry.strip() - if not stripped: - continue - - # Foreign key constraint - fk_match = _FOREIGN_KEY_RE.search(stripped) - if fk_match: - triples.extend( - self._extract_foreign_key( - fk_match, resource_uri, table_name, table_uri, schema_name - ) - ) - continue - - # Skip other constraints / non-column lines - first_word = ( - stripped.split()[0].strip('"').lower() if stripped.split() else "" - ) - if first_word in _NON_COLUMN_KEYWORDS: - continue - - # Column definition - col_match = _COLUMN_DEF_RE.match(stripped) - if col_match: - col_name = col_match.group(1) - col_type = col_match.group(2).strip() - - # Determine nullability and inline primary key - remainder = stripped[col_match.end() :] - is_not_null = bool(_NOT_NULL_RE.search(remainder)) - is_pk_inline = bool(_PRIMARY_KEY_INLINE_RE.search(remainder)) - is_pk = is_pk_inline or col_name.lower() in pk_columns - - triples.extend( - self._emit_column( - resource_uri, - table_name, - table_uri, - col_name, - col_type, - schema_name=schema_name, - is_nullable=not (is_not_null or is_pk), - is_primary_key=is_pk, - ) - ) - - return triples - - @staticmethod - def _split_entries(body: str) -> list[str]: - """Split a table body into entries by commas at depth 0.""" - entries: list[str] = [] - depth = 0 - current: list[str] = [] - for ch in body: - if ch == "(": - depth += 1 - current.append(ch) - elif ch == ")": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - entries.append("".join(current)) - current = [] - else: - current.append(ch) - tail = "".join(current).strip() - if tail: - entries.append(tail) - return entries - - @staticmethod - def _emit_column( - resource_uri: str, - table_name: str, - table_uri: str, - col_name: str, - col_type: str, - *, - schema_name: str = "", - is_nullable: bool, - is_primary_key: bool, - ) -> list[UKOTriple]: - """Produce triples for a single column definition.""" - triples: list[UKOTriple] = [] - c_uri = _column_uri(resource_uri, table_name, col_name, schema_name) - - # Type declaration - triples.append( - UKOTriple( - subject_uri=c_uri, - predicate="rdf:type", - object_uri="uko-data:Column", - ) - ) - - # Label - triples.append( - UKOTriple( - subject_uri=c_uri, - predicate="rdfs:label", - object_value=col_name, - ) - ) - - # Containment - triples.append( - UKOTriple( - subject_uri=table_uri, - predicate="uko:contains", - object_uri=c_uri, - ) - ) - - # Data type - triples.append( - UKOTriple( - subject_uri=c_uri, - predicate="uko-data:dataType", - object_value=col_type, - ) - ) - - # Nullability - triples.append( - UKOTriple( - subject_uri=c_uri, - predicate="uko-data:isNullable", - object_value=str(is_nullable).lower(), - ) - ) - - # Primary key - triples.append( - UKOTriple( - subject_uri=c_uri, - predicate="uko-data:isPrimaryKey", - object_value=str(is_primary_key).lower(), - ) - ) - - return triples - - def _extract_foreign_key( - self, - fk_match: re.Match[str], - resource_uri: str, - table_name: str, - table_uri: str, - schema_name: str = "", - ) -> list[UKOTriple]: - """Extract triples for a ``FOREIGN KEY`` constraint. - - Per spec §42214-42217, ``uko-data:foreignKeyTo`` links - Column → Column (domain = uko-data:Column, range = uko-data:Column). - - Supports multi-column foreign keys: each source/target column pair - gets its own ``foreignKeyTo`` edge. - """ - triples: list[UKOTriple] = [] - - # Parse comma-separated column lists (supports multi-column FKs). - source_cols = [ - c.strip().strip('"') for c in fk_match.group(1).split(",") if c.strip() - ] - ref_schema = fk_match.group(2) or schema_name # inherit current - ref_table = fk_match.group(3) - target_cols = [ - c.strip().strip('"') for c in fk_match.group(4).split(",") if c.strip() - ] - - # Use the first source column for the FK URI (composite FKs share - # a single ForeignKey node). - first_col = source_cols[0] if source_cols else "_unknown_" - fk_uri_val = _fk_uri(resource_uri, table_name, first_col, schema_name) - - # FK type declaration - triples.append( - UKOTriple( - subject_uri=fk_uri_val, - predicate="rdf:type", - object_uri="uko-data:ForeignKey", - ) - ) - - # Containment — FK belongs to the table - triples.append( - UKOTriple( - subject_uri=table_uri, - predicate="uko:contains", - object_uri=fk_uri_val, - ) - ) - - # Spec-compliant Column→Column links (one per column pair). - for src_col, tgt_col in zip(source_cols, target_cols, strict=False): - source_col_uri = _column_uri(resource_uri, table_name, src_col, schema_name) - target_col_uri = _column_uri(resource_uri, ref_table, tgt_col, ref_schema) - triples.append( - UKOTriple( - subject_uri=source_col_uri, - predicate="uko-data:foreignKeyTo", - object_uri=target_col_uri, - ) - ) - - return triples + # -- View extraction ------------------------------------------------------ def _extract_views( self, @@ -629,17 +252,16 @@ class PostgreSQLAnalyzer: """Extract triples for ``CREATE VIEW`` statements. Emits ``uko-data:viewDefinition`` with the SQL body of the view - (spec §42263-42266). + (spec section 42263-42266). """ triples: list[UKOTriple] = [] - for match in _CREATE_VIEW_RE.finditer(content): - schema_name = match.group(1) or "" - schema_lower = schema_name.lower() if schema_name else "" - view_name = match.group(2) - v_uri = _view_uri(resource_uri, view_name, schema_lower) + for match in CREATE_VIEW_RE.finditer(content): + s_name = match.group(1) or "" + schema_lower = s_name.lower() if s_name else "" + v_name = match.group(2) + v_uri = view_uri(resource_uri, v_name, schema_lower) - # View type declaration triples.append( UKOTriple( subject_uri=v_uri, @@ -647,18 +269,14 @@ class PostgreSQLAnalyzer: object_uri="uko-data:View", ) ) - - # View label triples.append( UKOTriple( subject_uri=v_uri, predicate="rdfs:label", - object_value=view_name, + object_value=v_name, ) ) - # View definition — capture the SQL after AS up to the - # next semicolon (or end of content). as_start = match.end() semi_pos = content.find(";", as_start) view_sql = ( @@ -675,11 +293,8 @@ class PostgreSQLAnalyzer: ) ) - # Schema containment - if schema_name: - s_uri = self._ensure_schema( - schema_name, resource_uri, schemas_seen, triples - ) + if s_name: + s_uri = self._ensure_schema(s_name, resource_uri, schemas_seen, triples) triples.append( UKOTriple( subject_uri=s_uri,