From eb8bfdbee5f3d1791c19fbaa52f9e95cf0f540f5 Mon Sep 17 00:00:00 2001 From: Brent Edwards Date: Tue, 23 Dec 2025 15:07:41 -0800 Subject: [PATCH] Adding the features file. --- features/environment.py | 79 ++ .../rdf_converter/cli_integration.feature | 82 ++ features/rdf_converter/error_handling.feature | 129 +++ features/rdf_converter/file_handling.feature | 288 ++++++ .../rdf_converter/parallel_conversion.feature | 162 ++++ .../rdf_converter/standard_conversion.feature | 126 +++ .../rdf_converter/strategy_selection.feature | 119 +++ .../streaming_conversion.feature | 172 ++++ features/steps/rdf_converter_steps.py | 880 ++++++++++++++++++ 9 files changed, 2037 insertions(+) create mode 100644 features/environment.py create mode 100644 features/rdf_converter/cli_integration.feature create mode 100644 features/rdf_converter/error_handling.feature create mode 100644 features/rdf_converter/file_handling.feature create mode 100644 features/rdf_converter/parallel_conversion.feature create mode 100644 features/rdf_converter/standard_conversion.feature create mode 100644 features/rdf_converter/strategy_selection.feature create mode 100644 features/rdf_converter/streaming_conversion.feature create mode 100644 features/steps/rdf_converter_steps.py diff --git a/features/environment.py b/features/environment.py new file mode 100644 index 0000000..65885af --- /dev/null +++ b/features/environment.py @@ -0,0 +1,79 @@ +"""Behave environment configuration with setup/teardown hooks. + +This module handles: +- Temporary directory creation and cleanup +- Shared test fixtures +- Logging configuration for tests +""" + +from __future__ import annotations + +import logging +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from behave.model import Feature, Scenario + from behave.runner import Context + + +# Suppress verbose logging from libraries during tests +logging.getLogger("rdflib").setLevel(logging.ERROR) +logging.getLogger("datasets").setLevel(logging.ERROR) +logging.getLogger("urllib3").setLevel(logging.ERROR) +logging.getLogger("filelock").setLevel(logging.ERROR) + + +def before_all(context: Context) -> None: + """Run once before all tests - create base temp directory.""" + context.base_temp_dir = Path(tempfile.mkdtemp(prefix="behave_rdf_tests_")) + context.cleanup_dirs = [] # list[Path] + + +def after_all(context: Context) -> None: + """Run once after all tests - cleanup base temp directory.""" + if hasattr(context, "base_temp_dir") and context.base_temp_dir.exists(): + shutil.rmtree(context.base_temp_dir, ignore_errors=True) + + # Cleanup any remaining registered directories + if hasattr(context, "cleanup_dirs"): + for cleanup_dir in context.cleanup_dirs: + if cleanup_dir.exists(): + shutil.rmtree(cleanup_dir, ignore_errors=True) + + +def before_feature(context: Context, feature: Feature) -> None: + """Run before each feature file.""" + # Create feature-specific temp directory + feature_name = feature.name.replace(" ", "_").lower()[:30] + context.feature_temp_dir = context.base_temp_dir / f"feature_{feature_name}" + context.feature_temp_dir.mkdir(parents=True, exist_ok=True) + + +def after_feature(context: Context, feature: Feature) -> None: + """Run after each feature file - cleanup feature temp directory.""" + if hasattr(context, "feature_temp_dir") and context.feature_temp_dir.exists(): + shutil.rmtree(context.feature_temp_dir, ignore_errors=True) + + +def before_scenario(context: Context, scenario: Scenario) -> None: + """Run before each scenario - create scenario-specific temp directory.""" + scenario_name = scenario.name.replace(" ", "_").lower()[:30] + context.scenario_temp_dir = context.feature_temp_dir / f"scenario_{scenario_name}" + context.scenario_temp_dir.mkdir(parents=True, exist_ok=True) + + # Initialize scenario-specific attributes + context.input_path = None + context.output_path = None + context.result = None + context.rdf_format = None + context.expected_triple_count = None + + +def after_scenario(context: Context, scenario: Scenario) -> None: + """Run after each scenario - cleanup scenario temp directory.""" + if hasattr(context, "scenario_temp_dir") and context.scenario_temp_dir.exists(): + shutil.rmtree(context.scenario_temp_dir, ignore_errors=True) + diff --git a/features/rdf_converter/cli_integration.feature b/features/rdf_converter/cli_integration.feature new file mode 100644 index 0000000..4184462 --- /dev/null +++ b/features/rdf_converter/cli_integration.feature @@ -0,0 +1,82 @@ +Feature: CLI Integration Tests + As a user running the converter from command line + I want the CLI to work correctly + So that I can convert RDF files without writing code + + Background: + Given a temporary output directory + + # ============================================ + # BASIC CLI EXECUTION + # ============================================ + + Scenario: CLI converts N-Triples file with auto strategy + Given an N-Triples file with 50 triples + When I run the CLI with auto strategy + Then the CLI should exit successfully + And the output directory should contain a dataset + + Scenario: CLI converts file with explicit standard strategy + Given an N-Triples file with 30 triples + When I run the CLI with strategy "standard" + Then the CLI should exit successfully + + Scenario: CLI converts file with streaming strategy + Given an N-Triples file with 100 triples + When I run the CLI with strategy "streaming" + Then the CLI should exit successfully + + Scenario: CLI with custom chunk size + Given an N-Triples file with 200 triples + When I run the CLI with chunk size 50 + Then the CLI should exit successfully + + # ============================================ + # CLI WITH METADATA + # ============================================ + + Scenario: CLI with description and license + Given an N-Triples file with 25 triples + When I run the CLI with description "Test" and license "MIT" + Then the CLI should exit successfully + + Scenario: CLI with all metadata flags + Given an N-Triples file with 20 triples + When I run the CLI with all metadata flags + Then the CLI should exit successfully + + # ============================================ + # CLI ERROR HANDLING + # ============================================ + + Scenario: CLI fails for non-existent input file + Given a non-existent input file path + When I run the CLI expecting failure + Then the CLI should exit with error + + Scenario: CLI fails for empty file and reports error + Given an empty N-Triples file + When I run the CLI expecting failure + Then the CLI should exit with error + + Scenario: CLI with verbose flag + Given an N-Triples file with 20 triples + When I run the CLI with verbose flag + Then the CLI should exit successfully + + Scenario: CLI with clean-cache flag + Given an N-Triples file with 30 triples + When I run the CLI with clean-cache flag + Then the CLI should exit successfully + + # ============================================ + # CLI WITH TRAIN/TEST SPLIT + # ============================================ + + Scenario: CLI creates train/test split + Given an N-Triples file with 500 triples + When I run the CLI with train/test split + Then the CLI should exit successfully + And the output should have train and test splits + + diff --git a/features/rdf_converter/error_handling.feature b/features/rdf_converter/error_handling.feature new file mode 100644 index 0000000..5a0fefb --- /dev/null +++ b/features/rdf_converter/error_handling.feature @@ -0,0 +1,129 @@ +Feature: Error Handling + As a data engineer + I want clear error messages when conversions fail + So that I can quickly diagnose and fix issues + + The converter should handle: + - Non-existent files + - Empty files + - Invalid RDF syntax + - Unsupported formats + + Background: + Given a temporary output directory + + # ============================================ + # FILE NOT FOUND + # ============================================ + + Scenario: Fail gracefully for non-existent file + Given a non-existent input file path + When I attempt conversion using the "standard" strategy + Then the conversion should fail + And the error message should contain "No such file" + + Scenario: Fail gracefully for non-existent file with streaming + Given a non-existent input file path + When I attempt conversion using the "streaming" strategy + Then the conversion should fail + + # ============================================ + # EMPTY FILES + # ============================================ + + Scenario: Handle empty N-Triples file + Given an empty N-Triples file + When I convert it using the "standard" strategy + Then the conversion should fail + And the error message should contain "No triples" + + Scenario: Handle empty Turtle file + Given an empty Turtle file + When I convert it using the "standard" strategy + Then the conversion should fail + + # ============================================ + # INVALID SYNTAX + # ============================================ + + Scenario: Handle malformed N-Triples gracefully + Given an N-Triples file containing + """ + "valid" . + this is not valid ntriples syntax + "also valid" . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 2 triples + + Scenario: Skip malformed lines in streaming mode + Given an N-Triples file containing + """ + "v1" . + broken line without proper format + "v2" . + another broken line + "v3" . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 3 triples + + # ============================================ + # RECOVERY + # ============================================ + + Scenario: Recover from partially valid file + Given an N-Triples file containing + """ + "value1" . + "value2" . + "value3" . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain at least 2 triples + + # ============================================ + # CONVERSION RESULT CONSISTENCY + # ============================================ + + Scenario: Failed conversion returns error details + Given an empty N-Triples file + When I convert it using the "standard" strategy + Then the conversion should fail + And the result should have an error message + + Scenario: Successful conversion has no error message + Given an N-Triples file with 10 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the result should not have an error message + + # ============================================ + # INVALID STRATEGY + # ============================================ + + Scenario: Invalid strategy name raises error + Given an N-Triples file with 10 triples + When I attempt conversion with invalid strategy "nonexistent" + Then a ValueError should be raised + + # ============================================ + # STREAMING ERROR RECOVERY + # ============================================ + + Scenario: Streaming recovers from parsing errors in ntriples + Given an N-Triples file containing + """ + "valid1" . + "invalid" . + "valid2" . + not a triple at all + "valid3" . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain at least 3 triples + diff --git a/features/rdf_converter/file_handling.feature b/features/rdf_converter/file_handling.feature new file mode 100644 index 0000000..32334e1 --- /dev/null +++ b/features/rdf_converter/file_handling.feature @@ -0,0 +1,288 @@ +Feature: File Format and Compression Handling + As a data engineer working with various RDF sources + I want to convert files in different formats and compressions + So that I can handle real-world data without preprocessing + + The converter supports: + - Formats: N-Triples, Turtle, RDF/XML, N3, TriG, NQuads, TSV + - Compression: gzip (.gz), bzip2 (.bz2) + - Automatic format and compression detection + + Background: + Given a temporary output directory + + # ============================================ + # N-TRIPLES FORMAT + # ============================================ + + Scenario: Convert uncompressed N-Triples file + Given an N-Triples file with 100 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 100 triples + + Scenario: N-Triples with URI objects + Given an N-Triples file containing + """ + . + . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 2 triples + And uri objects should have object_type as uri + + Scenario: N-Triples with literal objects + Given an N-Triples file containing + """ + "Alice" . + "Bob" . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And literal objects should have object_type as literal + + Scenario: N-Triples with language-tagged literals + Given an N-Triples file containing + """ + "Katze"@de . + "猫"@ja . + "кошка"@ru . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + + Scenario: N-Triples with typed literals + Given an N-Triples file containing + """ + "25"^^ . + "98.6"^^ . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 2 triples + + # ============================================ + # TURTLE FORMAT + # ============================================ + + Scenario: Convert Turtle file with prefixes + Given a Turtle file with 50 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 50 triples + + Scenario: Turtle with complex prefix declarations + Given a Turtle file containing + """ + @prefix ex: . + @prefix foaf: . + @prefix xsd: . + + ex:alice foaf:name "Alice" ; + foaf:age "30"^^xsd:integer . + ex:bob foaf:name "Bob" ; + foaf:knows ex:alice . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 4 triples + + Scenario: Turtle with streaming-turtle strategy + Given a Turtle file with 200 triples + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 200 triples + + # ============================================ + # GZIP COMPRESSION + # ============================================ + + Scenario: Convert gzip-compressed N-Triples + Given a gzip-compressed N-Triples file with 150 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 150 triples + + Scenario: Stream gzip-compressed N-Triples + Given a gzip-compressed N-Triples file with 200 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 200 triples + + Scenario: Parallel process gzip-compressed N-Triples + Given a gzip-compressed N-Triples file with 250 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 250 triples + + # ============================================ + # BZ2 COMPRESSION + # ============================================ + + Scenario: Convert bz2-compressed N-Triples + Given a bz2-compressed N-Triples file with 100 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 100 triples + + Scenario: Stream bz2-compressed N-Triples + Given a bz2-compressed N-Triples file with 150 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 150 triples + + # ============================================ + # TSV FORMAT + # ============================================ + + Scenario: Convert TSV file with standard strategy + Given a TSV file with 100 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 100 triples + And the output should have the correct schema + + Scenario: TSV file creates literal object types + Given a TSV file with 50 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And all triples should have valid object_type values + + Scenario: TSV file with empty lines and malformed rows + Given a TSV file with empty lines and malformed rows + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + + # ============================================ + # RDF/XML FORMAT + # ============================================ + + Scenario: Convert RDF/XML file with standard strategy + Given an RDF/XML file with 50 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 50 triples + And the output should have the correct schema + + # ============================================ + # GEONAMES FORMAT + # ============================================ + + Scenario: Convert GeoNames format file with standard strategy + Given a GeoNames format file with 10 features + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Convert GeoNames format file with streaming strategy + Given a GeoNames format file with 20 features + When I convert it using the "streaming" strategy + Then the conversion should succeed + + Scenario: Convert GeoNames format file with parallel strategy + Given a GeoNames format file with 15 features + When I convert it using the "streaming-parallel" strategy with 2 workers + Then the conversion should succeed + + Scenario: Convert GeoNames format file with simple streaming + Given a GeoNames format file with 10 features + When I convert it using the "streaming-simple" strategy + Then the conversion should succeed + + # ============================================ + # BLANK NODES + # ============================================ + + Scenario: Handle blank nodes in N-Triples + Given an N-Triples file containing + """ + _:b1 "Anonymous" . + _:b1 . + _:b2 "Thing" . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + And blank node objects should have object_type as blank_node + + Scenario: Handle blank nodes in Turtle + Given a Turtle file containing + """ + @prefix ex: . + + ex:person1 ex:address [ + ex:street "123 Main St" ; + ex:city "Springfield" + ] . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + + # ============================================ + # TURTLE EDGE CASES + # ============================================ + + Scenario: Turtle with @base declaration + Given a Turtle file containing + """ + @base . + @prefix foaf: . + + foaf:name "Alice" . + foaf:name "Bob" . + """ + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 2 triples + + Scenario: Turtle streaming with very small chunks + Given a Turtle file with 50 triples + And chunk size is set to 5 + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 50 triples + + Scenario: Turtle with special characters in literals + Given a Turtle file containing + """ + @prefix ex: . + + ex:doc ex:content "This has special chars: quotes \\\"test\\\" and newline \\n here" . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 1 triples + + # ============================================ + # EDGE CASES + # ============================================ + + Scenario: Handle N-Triples with blank lines + Given an N-Triples file containing + """ + "v1" . + + "v2" . + + "v3" . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + + Scenario: Handle N-Triples with comments + Given an N-Triples file containing + """ + # This is a comment + "v1" . + # Another comment + "v2" . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 2 triples + + diff --git a/features/rdf_converter/parallel_conversion.feature b/features/rdf_converter/parallel_conversion.feature new file mode 100644 index 0000000..806dae1 --- /dev/null +++ b/features/rdf_converter/parallel_conversion.feature @@ -0,0 +1,162 @@ +Feature: Parallel Streaming RDF to HuggingFace Conversion + As a data engineer processing very large RDF files + I want to convert RDF files using parallel multiprocessing + So that I can maximize throughput on multi-core systems + + The parallel streaming strategy uses multiprocessing.Pool to + distribute parsing work across multiple CPU cores. This is + optimal for files over 1GB on systems with multiple cores. + + Background: + Given a temporary output directory + + # ============================================ + # BASIC PARALLEL CONVERSION + # ============================================ + + Scenario: Convert N-Triples file using parallel strategy + Given an N-Triples file with 500 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 500 triples + And the output should have the correct schema + + Scenario: Convert Turtle file using parallel strategy + Given a Turtle file with 300 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 300 triples + + # ============================================ + # WORKER CONFIGURATION + # ============================================ + + Scenario: Parallel conversion with 2 workers + Given an N-Triples file with 400 triples + When I convert it using the "streaming-parallel" strategy with 2 workers + Then the conversion should succeed + And the output should contain 400 triples + And the output should have the correct schema + + Scenario: Parallel conversion with 4 workers + Given an N-Triples file with 600 triples + When I convert it using the "streaming-parallel" strategy with 4 workers + Then the conversion should succeed + And the output should contain 600 triples + + Scenario: Single worker behaves like serial processing + Given an N-Triples file with 200 triples + When I convert it using the "streaming-parallel" strategy with 1 workers + Then the conversion should succeed + And the output should contain 200 triples + + # ============================================ + # CHUNK SIZE CONFIGURATION + # ============================================ + + Scenario: Parallel with small chunk size creates more batches + Given an N-Triples file with 500 triples + And chunk size is set to 50 + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain the expected number of triples + + Scenario: Parallel with large chunk size + Given an N-Triples file with 300 triples + And chunk size is set to 500 + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 300 triples + + # ============================================ + # SCHEMA AND DATA INTEGRITY + # ============================================ + + Scenario: Parallel output has all required schema columns + Given an N-Triples file with 100 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Parallel preserves object type metadata + Given an N-Triples file with 150 triples and mixed object types + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And all triples should have valid object_type values + + Scenario: Parallel conversion produces same results as serial + Given an N-Triples file with 300 triples + When I convert it using the "streaming-parallel" strategy with 2 workers + Then the conversion should succeed + And the output should contain 300 triples + And the output should have the correct schema + + # ============================================ + # DATASET INFO + # ============================================ + + Scenario: Parallel conversion creates dataset_info.json + Given an N-Triples file with 200 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the dataset_info.json file should exist + And the dataset_info.json should contain total_triples + + Scenario: Parallel dataset_info.json includes worker count + Given an N-Triples file with 100 triples + When I convert it using the "streaming-parallel" strategy with 2 workers + Then the conversion should succeed + And the dataset_info.json file should exist + And the dataset_info.json should contain num_workers + + # ============================================ + # PROCESSING METRICS + # ============================================ + + Scenario: Parallel conversion records processing time + Given an N-Triples file with 400 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And processing time should be recorded + + # ============================================ + # LITERAL HANDLING + # ============================================ + + Scenario: Parallel correctly processes literals with language tags + Given an N-Triples file containing + """ + "hello"@en . + "hola"@es . + "ciao"@it . + "olá"@pt . + """ + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 4 triples + And literal objects should have object_type as literal + + Scenario: Parallel correctly processes URI objects + Given an N-Triples file containing + """ + . + . + . + """ + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 3 triples + And uri objects should have object_type as uri + + # ============================================ + # PARALLEL TURTLE CONVERSION + # ============================================ + + Scenario: Parallel processes Turtle file with chunking + Given a Turtle file with 200 triples + And chunk size is set to 50 + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 200 triples + + diff --git a/features/rdf_converter/standard_conversion.feature b/features/rdf_converter/standard_conversion.feature new file mode 100644 index 0000000..4aeb055 --- /dev/null +++ b/features/rdf_converter/standard_conversion.feature @@ -0,0 +1,126 @@ +Feature: Standard In-Memory RDF to HuggingFace Conversion + As a data engineer processing small RDF files + I want to convert RDF files using in-memory processing + So that I get fast, simple conversions for files under 100MB + + The standard strategy loads the entire RDF graph into memory, + converts all triples at once, and creates the dataset directly. + This is optimal for files under 100MB. + + Background: + Given a temporary output directory + + # ============================================ + # BASIC STANDARD CONVERSION + # ============================================ + + Scenario: Convert N-Triples file using standard strategy + Given an N-Triples file with 100 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 100 triples + And the output should have the correct schema + + Scenario: Convert Turtle file using standard strategy + Given a Turtle file with 150 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 150 triples + And the output should have the correct schema + + Scenario: Convert small file quickly with standard strategy + Given an N-Triples file with 50 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And processing time should be recorded + + # ============================================ + # TRAIN/TEST SPLIT + # ============================================ + + Scenario: Standard conversion with train/test split + Given an N-Triples file with 1000 triples + When I convert it using the "standard" strategy with train/test split + Then the conversion should succeed + And the output should have train and test splits + + Scenario: Standard conversion creates data split by default + Given an N-Triples file with 200 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should have a data split + + # ============================================ + # SCHEMA VALIDATION + # ============================================ + + Scenario: Standard output has all required schema columns + Given an N-Triples file with 25 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Standard preserves object type metadata + Given an N-Triples file with 60 triples and mixed object types + When I convert it using the "standard" strategy + Then the conversion should succeed + And all triples should have valid object_type values + + # ============================================ + # DATASET INFO + # ============================================ + + Scenario: Standard conversion creates dataset_info.json + Given an N-Triples file with 75 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the dataset_info.json file should exist + And the dataset_info.json should contain total_triples + + # ============================================ + # METADATA HANDLING + # ============================================ + + Scenario: Standard conversion with metadata + Given an N-Triples file with 50 triples + And metadata with description "Test dataset" and license "MIT" + When I convert it using the "standard" strategy with metadata + Then the conversion should succeed + And the dataset should have description metadata + + Scenario: Metadata with citation and homepage + Given an N-Triples file with 30 triples + And full metadata with all fields + When I convert it using the "standard" strategy with metadata + Then the conversion should succeed + And the dataset should have all metadata fields + + # ============================================ + # LITERAL HANDLING + # ============================================ + + Scenario: Standard correctly processes literals with datatypes + Given an N-Triples file containing + """ + "42"^^ . + "3.14"^^ . + "true"^^ . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + And literal objects should have object_type as literal + + Scenario: Standard correctly processes URI objects + Given an N-Triples file containing + """ + . + . + . + """ + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 3 triples + And uri objects should have object_type as uri + + diff --git a/features/rdf_converter/strategy_selection.feature b/features/rdf_converter/strategy_selection.feature new file mode 100644 index 0000000..7529eaf --- /dev/null +++ b/features/rdf_converter/strategy_selection.feature @@ -0,0 +1,119 @@ +Feature: Automatic Strategy Selection + As a data engineer + I want the converter to automatically select the best strategy + So that I get optimal performance without manual configuration + + The auto-selection logic considers: + - File size (< 100MB: standard, 100MB-1GB: streaming, > 1GB: parallel) + - RDF format (Turtle files may use streaming-turtle) + - GeoNames format detection + - Available CPU cores + + Background: + Given a temporary output directory + + # ============================================ + # SMALL FILE SELECTION (< 100MB) + # ============================================ + + Scenario: Auto selects standard strategy for small N-Triples file + Given an N-Triples file with 50 triples + When I convert it using the "auto" strategy + Then the conversion should succeed + And the output should contain 50 triples + And the output should have the correct schema + + Scenario: Auto selects strategy for small Turtle file + Given a Turtle file with 100 triples + When I convert it using the "auto" strategy + Then the conversion should succeed + And the output should contain 100 triples + + # ============================================ + # FORMAT DETECTION + # ============================================ + + Scenario: Auto handles N-Triples format correctly + Given an N-Triples file with 75 triples + When I convert it using the "auto" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Auto handles Turtle format correctly + Given a Turtle file with 80 triples + When I convert it using the "auto" strategy + Then the conversion should succeed + And the output should have the correct schema + + # ============================================ + # MANUAL STRATEGY OVERRIDE + # ============================================ + + Scenario: Can override auto with manual standard selection + Given an N-Triples file with 200 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should contain 200 triples + + Scenario: Can override auto with manual streaming selection + Given an N-Triples file with 150 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 150 triples + + Scenario: Can override auto with manual parallel selection + Given an N-Triples file with 250 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should contain 250 triples + + Scenario: Can override auto with streaming-simple selection + Given an N-Triples file with 100 triples + When I convert it using the "streaming-simple" strategy + Then the conversion should succeed + And the output should contain 100 triples + + Scenario: Can override auto with streaming-turtle selection + Given a Turtle file with 120 triples + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 120 triples + + # ============================================ + # STRATEGY PRODUCES CONSISTENT RESULTS + # ============================================ + + Scenario: All strategies produce correct schema + Given an N-Triples file with 100 triples + When I convert it using the "standard" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Streaming strategy produces correct schema + Given an N-Triples file with 100 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Parallel strategy produces correct schema + Given an N-Triples file with 100 triples + When I convert it using the "streaming-parallel" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Simple streaming produces correct schema + Given an N-Triples file with 100 triples + When I convert it using the "streaming-simple" strategy + Then the conversion should succeed + And the output should have the correct schema + + # ============================================ + # STRATEGY LISTING + # ============================================ + + Scenario: List all available strategies + When I list all available strategies + Then I should get 5 strategies + And strategies should include "standard, streaming" + + diff --git a/features/rdf_converter/streaming_conversion.feature b/features/rdf_converter/streaming_conversion.feature new file mode 100644 index 0000000..81c1e8f --- /dev/null +++ b/features/rdf_converter/streaming_conversion.feature @@ -0,0 +1,172 @@ +Feature: Streaming RDF to HuggingFace Conversion + As a data engineer processing medium to large RDF files + I want to convert RDF files using chunked streaming + So that I can process files that don't fit in memory + + The streaming strategy processes files in configurable chunks, + writes intermediate Parquet files, and merges them into a final dataset. + This is optimal for files between 100MB and 1GB. + + Background: + Given a temporary output directory + + # ============================================ + # BASIC STREAMING CONVERSION + # ============================================ + + Scenario: Convert N-Triples file using streaming strategy + Given an N-Triples file with 500 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 500 triples + And the output should have the correct schema + + Scenario: Convert Turtle file using streaming strategy + Given a Turtle file with 200 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 200 triples + And the output should have the correct schema + + Scenario: Streaming conversion with small chunk size + Given an N-Triples file with 500 triples + And chunk size is set to 50 + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain the expected number of triples + + # ============================================ + # TRAIN/TEST SPLIT + # ============================================ + + Scenario: Streaming conversion creates train/test splits when requested + Given an N-Triples file with 1000 triples + When I convert it using the "streaming" strategy with train/test split + Then the conversion should succeed + And the output should have train and test splits + + Scenario: Streaming conversion creates data split by default + Given an N-Triples file with 100 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should have a data split + + # ============================================ + # COMPRESSION SUPPORT + # ============================================ + + Scenario: Stream gzip-compressed N-Triples file + Given a gzip-compressed N-Triples file with 300 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 300 triples + + # ============================================ + # SCHEMA VALIDATION + # ============================================ + + Scenario: Streaming output has all required schema columns + Given an N-Triples file with 50 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should have the correct schema + + Scenario: Streaming preserves object type metadata + Given an N-Triples file with 100 triples and mixed object types + When I convert it using the "streaming" strategy + Then the conversion should succeed + And all triples should have valid object_type values + + # ============================================ + # DATASET INFO JSON + # ============================================ + + Scenario: Streaming conversion creates dataset_info.json + Given an N-Triples file with 100 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the dataset_info.json file should exist + And the dataset_info.json should contain total_triples + + # ============================================ + # PROCESSING METRICS + # ============================================ + + Scenario: Streaming conversion records processing time + Given an N-Triples file with 200 triples + When I convert it using the "streaming" strategy + Then the conversion should succeed + And processing time should be recorded + + # ============================================ + # EDGE CASES + # ============================================ + + Scenario: Streaming handles file with exactly one chunk + Given an N-Triples file with 50 triples + And chunk size is set to 100 + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 50 triples + + Scenario: Streaming handles file smaller than chunk size + Given an N-Triples file with 10 triples + And chunk size is set to 1000 + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 10 triples + + # ============================================ + # LITERAL HANDLING + # ============================================ + + Scenario: Streaming correctly processes literals with language tags + Given an N-Triples file containing + """ + "hello"@en . + "bonjour"@fr . + "hallo"@de . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 3 triples + And literal objects should have object_type as literal + + Scenario: Streaming correctly processes URI objects + Given an N-Triples file containing + """ + . + . + """ + When I convert it using the "streaming" strategy + Then the conversion should succeed + And the output should contain 2 triples + And uri objects should have object_type as uri + + # ============================================ + # STREAMING TURTLE WITH MULTILINE + # ============================================ + + Scenario: Streaming-turtle handles multiline statements + Given a Turtle file with multiline statements + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 2 triples + + Scenario: Streaming-turtle with small chunk triggers chunking + Given a Turtle file with 100 triples + And chunk size is set to 10 + When I convert it using the "streaming-turtle" strategy + Then the conversion should succeed + And the output should contain 100 triples + + # ============================================ + # STREAMING WITHOUT CLEAN CACHE + # ============================================ + + Scenario: Streaming conversion without clean cache + Given an N-Triples file with 100 triples + When I convert it using streaming without clean cache + Then the conversion should succeed + And the output should contain 100 triples + diff --git a/features/steps/rdf_converter_steps.py b/features/steps/rdf_converter_steps.py new file mode 100644 index 0000000..fccc02f --- /dev/null +++ b/features/steps/rdf_converter_steps.py @@ -0,0 +1,880 @@ +"""Step definitions for RDF to HuggingFace converter tests. + +This module provides reusable step definitions for testing: +- All conversion strategies (standard, streaming, parallel, etc.) +- File format handling (N-Triples, Turtle, RDF/XML, etc.) +- Compression support (gzip, bz2) +- Schema validation +- Error handling +""" + +from __future__ import annotations + +import bz2 +import gzip +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from behave import given, then, when + +# Add scripts directory to path for imports +SCRIPTS_DIR = Path(__file__).parent.parent.parent / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +from convert_rdf_to_hf_dataset_unified import ( + ConversionConfig, + ConversionResult, + FileHandler, + ProgressTracker, + SchemaManager, + StrategySelector, +) + +if TYPE_CHECKING: + from behave.runner import Context + +# HELPER FUNCTIONS + +def _create_ntriples_content(count: int, include_types: bool = False) -> str: + """Generate N-Triples content with specified number of triples.""" + lines = [] + for i in range(count): + if include_types and i % 3 == 0: + # URI object + lines.append( + f" " + f" " + f" ." + ) + elif include_types and i % 3 == 1: + # Literal with language + lines.append( + f" " + f" " + f'"value{i}"@en .' + ) + else: + # Plain literal + lines.append( + f" " + f" " + f'"value{i}" .' + ) + return "\n".join(lines) + + +def _create_turtle_content(count: int) -> str: + """Generate Turtle content with prefixes and specified number of triples.""" + lines = [ + "@prefix ex: .", + "@prefix xsd: .", + "", + ] + for i in range(count): + lines.append(f'ex:s{i} ex:p "value{i}" .') + return "\n".join(lines) + + +def _get_strategy(context: Context, strategy_name: str): + """Get a conversion strategy by name.""" + progress = ProgressTracker(verbose=False) + selector = StrategySelector(progress) + + if strategy_name == "auto": + return selector.select_auto(context.input_path, context.rdf_format) + return selector.select_manual(strategy_name) + + +def _run_conversion( + context: Context, + strategy_name: str, + num_workers: int | None = None, + chunk_size: int = 1000, + create_splits: bool = False, +) -> ConversionResult: + """Execute conversion and store result in context.""" + strategy = _get_strategy(context, strategy_name) + + config = ConversionConfig( + input_path=context.input_path, + output_path=context.output_path, + rdf_format=context.rdf_format, + chunk_size=chunk_size, + num_workers=num_workers, + create_train_test_split=create_splits, + clean_cache=True, + ) + + return strategy.convert(config) + + +def _run_conversion_with_metadata( + context: Context, + strategy_name: str, + num_workers: int | None = None, + chunk_size: int = 1000, + metadata: dict | None = None, +) -> ConversionResult: + """Execute conversion with metadata.""" + strategy = _get_strategy(context, strategy_name) + + config = ConversionConfig( + input_path=context.input_path, + output_path=context.output_path, + rdf_format=context.rdf_format, + chunk_size=chunk_size, + num_workers=num_workers, + metadata=metadata, + clean_cache=True, + ) + + return strategy.convert(config) + + +def _load_output_dataset(context: Context): + """Load the output dataset from disk.""" + from datasets import load_from_disk + + dataset_dict = load_from_disk(str(context.output_path)) + + # Get the actual dataset (could be 'data', 'train', etc.) + if "data" in dataset_dict: + return dataset_dict, dataset_dict["data"] + if "train" in dataset_dict: + return dataset_dict, dataset_dict["train"] + return dataset_dict, list(dataset_dict.values())[0] + + +# ============================================================================= +# GIVEN STEPS - Test Setup +# ============================================================================= + + +@given("a temporary output directory") +def step_create_temp_output_dir(context: Context) -> None: + """Create a temporary output directory for the conversion.""" + context.output_path = context.scenario_temp_dir / "output" + context.output_path.mkdir(parents=True, exist_ok=True) + + +@given("an N-Triples file with {count:d} triples") +def step_create_ntriples_file(context: Context, count: int) -> None: + """Create a test N-Triples file with the specified number of triples.""" + context.input_path = context.scenario_temp_dir / "test.nt" + content = _create_ntriples_content(count) + context.input_path.write_text(content) + context.expected_triple_count = count + context.rdf_format = "nt" + + +@given("an N-Triples file with {count:d} triples and mixed object types") +def step_create_mixed_ntriples_file(context: Context, count: int) -> None: + """Create N-Triples with URIs, literals with language tags, and plain literals.""" + context.input_path = context.scenario_temp_dir / "test_mixed.nt" + content = _create_ntriples_content(count, include_types=True) + context.input_path.write_text(content) + context.expected_triple_count = count + context.rdf_format = "nt" + + +@given("a Turtle file with {count:d} triples") +def step_create_turtle_file(context: Context, count: int) -> None: + """Create a test Turtle file with prefixes and specified number of triples.""" + context.input_path = context.scenario_temp_dir / "test.ttl" + content = _create_turtle_content(count) + context.input_path.write_text(content) + context.expected_triple_count = count + context.rdf_format = "turtle" + + +@given("a gzip-compressed N-Triples file with {count:d} triples") +def step_create_gzipped_ntriples(context: Context, count: int) -> None: + """Create a gzip-compressed N-Triples file.""" + context.input_path = context.scenario_temp_dir / "test.nt.gz" + content = _create_ntriples_content(count) + with gzip.open(context.input_path, "wt", encoding="utf-8") as f: + f.write(content) + context.expected_triple_count = count + context.rdf_format = "nt" + + +@given("an N-Triples file containing") +def step_create_ntriples_from_docstring(context: Context) -> None: + """Create an N-Triples file from the scenario's docstring content.""" + context.input_path = context.scenario_temp_dir / "test_custom.nt" + context.input_path.write_text(context.text.strip()) + context.rdf_format = "nt" + # Count non-empty, non-comment lines + lines = [ + line + for line in context.text.strip().split("\n") + if line.strip() and not line.strip().startswith("#") + ] + context.expected_triple_count = len(lines) + + +@given("a Turtle file containing") +def step_create_turtle_from_docstring(context: Context) -> None: + """Create a Turtle file from the scenario's docstring content.""" + context.input_path = context.scenario_temp_dir / "test_custom.ttl" + context.input_path.write_text(context.text.strip()) + context.rdf_format = "turtle" + + +@given("chunk size is set to {size:d}") +def step_set_chunk_size(context: Context, size: int) -> None: + """Set the chunk size for streaming conversion.""" + context.chunk_size = size + + +@given("a bz2-compressed N-Triples file with {count:d} triples") +def step_create_bz2_ntriples(context: Context, count: int) -> None: + """Create a bz2-compressed N-Triples file.""" + context.input_path = context.scenario_temp_dir / "test.nt.bz2" + content = _create_ntriples_content(count) + with bz2.open(context.input_path, "wt", encoding="utf-8") as f: + f.write(content) + context.expected_triple_count = count + context.rdf_format = "nt" + + +@given("a TSV file with {count:d} triples") +def step_create_tsv_file(context: Context, count: int) -> None: + """Create a TSV file with subject/predicate/object columns.""" + context.input_path = context.scenario_temp_dir / "test.tsv" + lines = [] + for i in range(count): + lines.append(f"entity{i}\trelation{i % 10}\tentity{i + 1}") + context.input_path.write_text("\n".join(lines)) + context.expected_triple_count = count + context.rdf_format = "tsv" + + +@given("a non-existent input file path") +def step_set_nonexistent_file(context: Context) -> None: + """Set input path to a non-existent file.""" + context.input_path = context.scenario_temp_dir / "does_not_exist.nt" + context.rdf_format = "nt" + + +@given("an empty N-Triples file") +def step_create_empty_ntriples(context: Context) -> None: + """Create an empty N-Triples file.""" + context.input_path = context.scenario_temp_dir / "empty.nt" + context.input_path.write_text("") + context.rdf_format = "nt" + + +@given("an empty Turtle file") +def step_create_empty_turtle(context: Context) -> None: + """Create an empty Turtle file.""" + context.input_path = context.scenario_temp_dir / "empty.ttl" + context.input_path.write_text("") + context.rdf_format = "turtle" + + +@given("an RDF/XML file with {count:d} triples") +def step_create_rdfxml_file(context: Context, count: int) -> None: + """Create an RDF/XML file with specified number of triples.""" + context.input_path = context.scenario_temp_dir / "test.rdf" + triples_xml = "\n".join( + f' \n' + f' value{i}\n' + f' ' + for i in range(count) + ) + content = f''' + +{triples_xml} +''' + context.input_path.write_text(content) + context.expected_triple_count = count + context.rdf_format = "xml" + + +@given('metadata with description "{desc}" and license "{lic}"') +def step_set_metadata(context: Context, desc: str, lic: str) -> None: + """Set metadata for the conversion.""" + context.metadata = {"description": desc, "license": lic} + + +@given("full metadata with all fields") +def step_set_full_metadata(context: Context) -> None: + """Set metadata with all fields: description, citation, homepage, license.""" + context.metadata = { + "description": "Test dataset description", + "citation": "@misc{test2024, title={Test}}", + "homepage": "https://example.org/dataset", + "license": "MIT", + } + + +@given("a TSV file with empty lines and malformed rows") +def step_create_tsv_with_issues(context: Context) -> None: + """Create a TSV file with empty lines and rows that don't have 3 columns.""" + context.input_path = context.scenario_temp_dir / "test_issues.tsv" + content = """entity1\trelation1\tentity2 + +entity3\trelation3\tentity4 +malformed_row_only_two_columns\tvalue +entity5\trelation5\tentity6 +single_column +""" + context.input_path.write_text(content) + context.expected_triple_count = 3 + context.rdf_format = "tsv" + + +@given("a Turtle file with multiline statements") +def step_create_turtle_multiline(context: Context) -> None: + """Create a Turtle file with statements spanning multiple lines.""" + context.input_path = context.scenario_temp_dir / "multiline.ttl" + content = """@prefix ex: . +@prefix foaf: . + +ex:person1 foaf:name "Alice" ; + foaf:age "30" . +""" + context.input_path.write_text(content) + context.expected_triple_count = 2 + context.rdf_format = "turtle" + + +@given("a GeoNames format file with {count:d} features") +def step_create_geonames_file(context: Context, count: int) -> None: + """Create a GeoNames-style RDF/XML file with URL prefixes.""" + context.input_path = context.scenario_temp_dir / "geonames_test.xml" + + docs = [] + for i in range(count): + docs.append(f"""http://sws.geonames.org/{1000000 + i}/ + + + + Location{i} + +""") + + context.input_path.write_text("\n".join(docs)) + context.expected_triple_count = count + context.rdf_format = "xml" + + +# ============================================================================= +# WHEN STEPS - Execute Actions +# ============================================================================= + + +@when('I convert it using the "{strategy_name}" strategy') +def step_convert_with_strategy(context: Context, strategy_name: str) -> None: + """Execute conversion with the specified strategy.""" + chunk_size = getattr(context, "chunk_size", 1000) + context.result = _run_conversion(context, strategy_name, chunk_size=chunk_size) + context.strategy_used = strategy_name + + +@when('I convert it using the "{strategy_name}" strategy with {workers:d} workers') +def step_convert_with_workers(context: Context, strategy_name: str, workers: int) -> None: + """Execute conversion with specified strategy and worker count.""" + chunk_size = getattr(context, "chunk_size", 1000) + context.result = _run_conversion( + context, strategy_name, num_workers=workers, chunk_size=chunk_size + ) + context.strategy_used = strategy_name + context.num_workers_used = workers + + +@when('I convert it using the "{strategy_name}" strategy with train/test split') +def step_convert_with_split(context: Context, strategy_name: str) -> None: + """Execute conversion with train/test split enabled.""" + chunk_size = getattr(context, "chunk_size", 1000) + context.result = _run_conversion( + context, strategy_name, chunk_size=chunk_size, create_splits=True + ) + context.strategy_used = strategy_name + + +@when("I convert it using streaming without clean cache") +def step_convert_streaming_no_clean(context: Context) -> None: + """Execute streaming conversion without clean_cache flag.""" + strategy = _get_strategy(context, "streaming") + config = ConversionConfig( + input_path=context.input_path, + output_path=context.output_path, + rdf_format=context.rdf_format, + chunk_size=getattr(context, "chunk_size", 1000), + clean_cache=False, # This is the key difference + ) + context.result = strategy.convert(config) + context.strategy_used = "streaming" + + +@when('I attempt conversion using the "{strategy_name}" strategy') +def step_attempt_conversion(context: Context, strategy_name: str) -> None: + """Attempt conversion that may fail (e.g., file not found).""" + chunk_size = getattr(context, "chunk_size", 1000) + try: + context.result = _run_conversion(context, strategy_name, chunk_size=chunk_size) + except FileNotFoundError as e: + context.result = ConversionResult( + success=False, + error_message=f"File not found: {e}" + ) + except Exception as e: + context.result = ConversionResult( + success=False, + error_message=str(e) + ) + context.strategy_used = strategy_name + + +@when('I convert it using the "{strategy_name}" strategy with metadata') +def step_convert_with_metadata(context: Context, strategy_name: str) -> None: + """Execute conversion with metadata.""" + chunk_size = getattr(context, "chunk_size", 1000) + metadata = getattr(context, "metadata", None) + context.result = _run_conversion_with_metadata( + context, strategy_name, chunk_size=chunk_size, metadata=metadata + ) + context.strategy_used = strategy_name + + +@when('I attempt conversion with invalid strategy "{strategy_name}"') +def step_attempt_invalid_strategy(context: Context, strategy_name: str) -> None: + """Attempt conversion with an invalid strategy name.""" + context.raised_exception = None + try: + _get_strategy(context, strategy_name) + except ValueError as e: + context.raised_exception = e + + +@when("I list all available strategies") +def step_list_strategies(context: Context) -> None: + """List all available strategies.""" + progress = ProgressTracker(verbose=False) + selector = StrategySelector(progress) + context.strategies_list = selector.list_strategies() + + +# ============================================================================= +# THEN STEPS - Verify Outcomes +# ============================================================================= + + +@then("the conversion should succeed") +def step_verify_success(context: Context) -> None: + """Verify the conversion completed successfully.""" + assert context.result is not None, "No conversion result found" + assert context.result.success, ( + f"Conversion failed: {context.result.error_message}" + ) + + +@then("the conversion should fail") +def step_verify_failure(context: Context) -> None: + """Verify the conversion failed as expected.""" + assert context.result is not None, "No conversion result found" + assert not context.result.success, "Expected conversion to fail but it succeeded" + + +@then("the conversion should fail with error containing {error_text}") +def step_verify_failure_message(context: Context, error_text: str) -> None: + """Verify the conversion failed with a specific error message.""" + assert not context.result.success, "Expected conversion to fail" + assert error_text in (context.result.error_message or ""), ( + f"Expected error containing '{error_text}', " + f"got: {context.result.error_message}" + ) + + +@then("the output should contain {count:d} triples") +def step_verify_triple_count(context: Context, count: int) -> None: + """Verify the number of triples in the output dataset.""" + assert context.result.total_triples == count, ( + f"Expected {count} triples, got {context.result.total_triples}" + ) + + +@then("the output should contain the expected number of triples") +def step_verify_expected_triple_count(context: Context) -> None: + """Verify the output contains the expected number of triples from setup.""" + expected = context.expected_triple_count + assert context.result.total_triples == expected, ( + f"Expected {expected} triples, got {context.result.total_triples}" + ) + + +@then("the output should have the correct schema") +def step_verify_schema(context: Context) -> None: + """Verify the output dataset has the expected RDF triple schema.""" + _, dataset = _load_output_dataset(context) + + expected_columns = { + "subject", + "predicate", + "object", + "object_type", + "object_datatype", + "object_language", + } + actual_columns = set(dataset.column_names) + + assert expected_columns == actual_columns, ( + f"Schema mismatch. Expected {expected_columns}, got {actual_columns}" + ) + + +@then("the output should have a {split_name} split") +def step_verify_split_exists(context: Context, split_name: str) -> None: + """Verify the output has a specific split (train, test, data).""" + from datasets import load_from_disk + + dataset_dict = load_from_disk(str(context.output_path)) + assert split_name in dataset_dict, ( + f"Expected '{split_name}' split, found: {list(dataset_dict.keys())}" + ) + + +@then("the output should have train and test splits") +def step_verify_train_test_splits(context: Context) -> None: + """Verify the output has both train and test splits.""" + from datasets import load_from_disk + + dataset_dict = load_from_disk(str(context.output_path)) + assert "train" in dataset_dict, f"Missing 'train' split: {list(dataset_dict.keys())}" + assert "test" in dataset_dict, f"Missing 'test' split: {list(dataset_dict.keys())}" + + +@then("the dataset_info.json file should exist") +def step_verify_info_json_exists(context: Context) -> None: + """Verify the dataset_info.json file was created.""" + info_path = context.output_path / "dataset_info.json" + assert info_path.exists(), f"dataset_info.json not found at {info_path}" + + +@then("the dataset_info.json should contain total_triples") +def step_verify_info_has_total_triples(context: Context) -> None: + """Verify dataset_info.json contains total_triples field.""" + import json + + info_path = context.output_path / "dataset_info.json" + with open(info_path) as f: + info = json.load(f) + + assert "total_triples" in info, f"Missing 'total_triples' in info: {info.keys()}" + assert info["total_triples"] == context.result.total_triples + + +@then("processing time should be recorded") +def step_verify_processing_time(context: Context) -> None: + """Verify processing time was recorded.""" + assert context.result.processing_time_seconds > 0, "Processing time not recorded" + + +@then("all triples should have valid object_type values") +def step_verify_object_types(context: Context) -> None: + """Verify all triples have valid object_type (uri, literal, blank_node).""" + _, dataset = _load_output_dataset(context) + + valid_types = {"uri", "literal", "blank_node"} + for row in dataset: + obj_type = row["object_type"] + assert obj_type in valid_types, ( + f"Invalid object_type: {obj_type}. Expected one of {valid_types}" + ) + + +@then("literal objects should have object_type as literal") +def step_verify_literal_type(context: Context) -> None: + """Verify literal objects have correct object_type.""" + _, dataset = _load_output_dataset(context) + + for row in dataset: + if row["object"].startswith('"') or row["object_language"]: + assert row["object_type"] == "literal", ( + f"Expected object_type 'literal' for {row['object']}" + ) + + +@then("uri objects should have object_type as uri") +def step_verify_uri_type(context: Context) -> None: + """Verify URI objects have correct object_type.""" + _, dataset = _load_output_dataset(context) + + for row in dataset: + if row["object"].startswith("http://") or row["object"].startswith("https://"): + if row["object_type"] != "blank_node": + assert row["object_type"] == "uri", ( + f"Expected object_type 'uri' for {row['object']}" + ) + + +@then('the error message should contain "{text}"') +def step_verify_error_contains(context: Context, text: str) -> None: + """Verify error message contains expected text.""" + assert context.result is not None, "No conversion result found" + assert not context.result.success, "Expected conversion to fail" + error_msg = context.result.error_message or "" + assert text.lower() in error_msg.lower(), ( + f"Expected error containing '{text}', got: {error_msg}" + ) + + +@then("the result should have an error message") +def step_verify_has_error_message(context: Context) -> None: + """Verify the result has an error message.""" + assert context.result is not None, "No conversion result found" + assert context.result.error_message, "Expected an error message but found none" + + +@then("the result should not have an error message") +def step_verify_no_error_message(context: Context) -> None: + """Verify the result has no error message.""" + assert context.result is not None, "No conversion result found" + assert not context.result.error_message, ( + f"Expected no error message but got: {context.result.error_message}" + ) + + +@then("the output should contain at least {count:d} triples") +def step_verify_min_triple_count(context: Context, count: int) -> None: + """Verify the output contains at least the specified number of triples.""" + assert context.result.total_triples >= count, ( + f"Expected at least {count} triples, got {context.result.total_triples}" + ) + + +@then("the dataset_info.json should contain num_workers") +def step_verify_info_has_workers(context: Context) -> None: + """Verify dataset_info.json contains num_workers field.""" + import json + + info_path = context.output_path / "dataset_info.json" + with open(info_path) as f: + info = json.load(f) + + assert "num_workers" in info, f"Missing 'num_workers' in info: {info.keys()}" + + +@then("blank node objects should have object_type as blank_node") +def step_verify_blank_node_type(context: Context) -> None: + """Verify at least one triple has a blank_node object_type.""" + _, dataset = _load_output_dataset(context) + + blank_node_count = sum( + 1 for row in dataset if row["object_type"] == "blank_node" + ) + assert blank_node_count > 0, ( + f"Expected at least one blank_node object, found none. " + f"Object types: {[row['object_type'] for row in dataset]}" + ) + + +@then("the dataset should have description metadata") +def step_verify_description_metadata(context: Context) -> None: + """Verify the dataset has description metadata.""" + dataset_dict, dataset = _load_output_dataset(context) + info = dataset.info + assert info.description, "Dataset has no description metadata" + + +@then("the dataset should have all metadata fields") +def step_verify_all_metadata(context: Context) -> None: + """Verify the dataset has all metadata fields.""" + dataset_dict, dataset = _load_output_dataset(context) + info = dataset.info + assert isinstance(info.description, str) and info.description, "Missing or invalid description" + assert isinstance(info.citation, str) and info.citation, "Missing or invalid citation" + assert isinstance(info.homepage, str) and info.homepage, "Missing or invalid homepage" + assert isinstance(info.license, str) and info.license, "Missing or invalid license" + + +@then("a ValueError should be raised") +def step_verify_valueerror(context: Context) -> None: + """Verify a ValueError was raised.""" + assert context.raised_exception is not None, "No exception was raised" + assert isinstance(context.raised_exception, ValueError), ( + f"Expected ValueError, got {type(context.raised_exception)}" + ) + + +@then("I should get {count:d} strategies") +def step_verify_strategy_count(context: Context, count: int) -> None: + """Verify the number of strategies returned.""" + assert len(context.strategies_list) == count, ( + f"Expected {count} strategies, got {len(context.strategies_list)}" + ) + + +@then('strategies should include "{strategy_names}"') +def step_verify_strategies_include(context: Context, strategy_names: str) -> None: + """Verify specific strategies are in the list.""" + names = [s[0] for s in context.strategies_list] + expected_strategies = [s.strip() for s in strategy_names.split(",")] + for strategy in expected_strategies: + assert strategy in names, f"Strategy '{strategy}' not found in {names}" + + +# ============================================================================= +# CLI INTEGRATION STEPS +# ============================================================================= + +def _run_cli(args: list[str]) -> tuple[int, str, str]: + """Run the CLI by calling main() directly for coverage tracking.""" + import io + from contextlib import redirect_stdout, redirect_stderr + from convert_rdf_to_hf_dataset_unified import main + + old_argv = sys.argv + stdout_capture = io.StringIO() + stderr_capture = io.StringIO() + + try: + sys.argv = ["convert_rdf_to_hf_dataset_unified.py"] + args + with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): + try: + exit_code = main() + except SystemExit as e: + exit_code = e.code if e.code is not None else 0 + finally: + sys.argv = old_argv + + return exit_code, stdout_capture.getvalue(), stderr_capture.getvalue() + + +@when("I run the CLI with auto strategy") +def step_run_cli_auto(context: Context) -> None: + """Run CLI with auto strategy.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when('I run the CLI with strategy "{strategy}"') +def step_run_cli_strategy(context: Context, strategy: str) -> None: + """Run CLI with specified strategy.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--strategy", strategy, + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI with chunk size {size:d}") +def step_run_cli_chunk_size(context: Context, size: int) -> None: + """Run CLI with custom chunk size.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--chunk-size", str(size), + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when('I run the CLI with description "{desc}" and license "{lic}"') +def step_run_cli_metadata(context: Context, desc: str, lic: str) -> None: + """Run CLI with metadata options.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--description", desc, + "--license", lic, + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI with all metadata flags") +def step_run_cli_all_metadata(context: Context) -> None: + """Run CLI with all metadata flags: description, citation, homepage, license.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--description", "Test Dataset", + "--citation", "@misc{test}", + "--homepage", "https://example.org", + "--license", "MIT", + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI expecting failure") +def step_run_cli_expect_failure(context: Context) -> None: + """Run CLI expecting it to fail.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI with verbose flag") +def step_run_cli_verbose(context: Context) -> None: + """Run CLI with verbose flag.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--verbose", + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI with clean-cache flag") +def step_run_cli_clean_cache(context: Context) -> None: + """Run CLI with clean-cache flag.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--clean-cache", + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@when("I run the CLI with train/test split") +def step_run_cli_split(context: Context) -> None: + """Run CLI with train/test split option.""" + args = [ + str(context.input_path), + str(context.output_path), + "--format", context.rdf_format, + "--create-splits", + ] + context.cli_exit_code, context.cli_stdout, context.cli_stderr = _run_cli(args) + + +@then("the CLI should exit successfully") +def step_verify_cli_success(context: Context) -> None: + """Verify CLI exited with code 0.""" + assert context.cli_exit_code == 0, ( + f"CLI failed with exit code {context.cli_exit_code}.\n" + f"stdout: {context.cli_stdout}\nstderr: {context.cli_stderr}" + ) + + +@then("the CLI should exit with error") +def step_verify_cli_error(context: Context) -> None: + """Verify CLI exited with non-zero code.""" + assert context.cli_exit_code != 0, "Expected CLI to fail but it succeeded" + + +@then("the output directory should contain a dataset") +def step_verify_output_has_dataset(context: Context) -> None: + """Verify output directory has dataset files.""" + assert context.output_path.exists(), f"Output path doesn't exist: {context.output_path}" + files = list(context.output_path.iterdir()) + assert len(files) > 0, "Output directory is empty" +