feat: Update to new CleverSwarm ontology loading capabilities #9
@@ -3,3 +3,6 @@
|
||||
## v0.0.1
|
||||
* Initial release.
|
||||
|
||||
## v0.1.0
|
||||
* Allow single RDF ontoly submission (JSON ontology file is now optional)
|
||||
* Enable logging to STDOUT by default
|
||||
|
||||
@@ -7,6 +7,7 @@ addopts =
|
||||
--verbose
|
||||
--tb=short
|
||||
--cov=src/cleverswarm_python_client
|
||||
--cov-config=.coveragerc
|
||||
--cov-report=html
|
||||
--cov-report=term-missing
|
||||
--cov-fail-under=85
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[metadata]
|
||||
name = cleverswarm_python_client
|
||||
version = 0.0.1
|
||||
version = 0.1.0
|
||||
description = A set of Python client utilities and SDK for interacting with CleverSwarm micro-services
|
||||
long_description = file: README.md
|
||||
long_description_content_type = text/markdown
|
||||
|
||||
@@ -19,6 +19,7 @@ CleverSwarm - text to knowledge graph extraction benchmark client.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set
|
||||
@@ -282,13 +283,12 @@ class BenchmarkCLI(object):
|
||||
def main():
|
||||
"""Main entry point for the benchmark client console script."""
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("PYTHON_LOGGING_LEVEL", logging.INFO),
|
||||
stream=sys.stderr,
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
format=("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CleverSwarm benchmark client for REST API."
|
||||
|
||||
@@ -19,6 +19,7 @@ CleverSwarm - unstructured text to knowledge graph triplets extractor client.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
@@ -76,7 +77,10 @@ class TextoToKGCLI(object):
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
):
|
||||
input_text_path: Path = self._input_prefix / input_text
|
||||
ontology_json_path: Path = self._input_prefix / ontology_json
|
||||
# Ontology JSON file is optional - only create path if it was provided
|
||||
ontology_json_path: Path | None = (
|
||||
self._input_prefix / ontology_json if ontology_json else None
|
||||
)
|
||||
ontology_owl_path: Path = self._input_prefix / ontology_owl
|
||||
wildcards_path: Path = Path(wildcards) if wildcards else None
|
||||
|
||||
@@ -85,11 +89,19 @@ class TextoToKGCLI(object):
|
||||
f"Unstructured input text does not exist or is not a file: {str(input_text_path)}"
|
||||
)
|
||||
|
||||
if not ontology_json_path.exists() or not ontology_json_path.is_file():
|
||||
# If ontology_json was provided, validate that the file exists
|
||||
if (
|
||||
ontology_json
|
||||
and ontology_json_path
|
||||
and (not ontology_json_path.exists() or not ontology_json_path.is_file())
|
||||
):
|
||||
raise ClientException(
|
||||
f"Input ontology JSON file does not exist or is not a file: {str(ontology_json_path)}"
|
||||
f"Specified optional Ontology JSON file does not exist or is not a file: {str(ontology_json_path)}"
|
||||
)
|
||||
|
||||
# Ontology JSON file is optional - use the path if it was provided and exists
|
||||
ontology_json_path_final: Path | None = ontology_json_path
|
||||
|
||||
if not ontology_owl_path.exists() or not ontology_owl_path.is_file():
|
||||
raise ClientException(
|
||||
f"Input ontology OWL file does not exist or is not a file: {str(ontology_owl_path)}"
|
||||
@@ -106,7 +118,7 @@ class TextoToKGCLI(object):
|
||||
if wildcards_job:
|
||||
job_id: str = self._client.create_unstructured_to_kg_wildcards_job(
|
||||
input_text_path,
|
||||
ontology_json_path,
|
||||
ontology_json_path_final,
|
||||
ontology_owl_path,
|
||||
wildcards_path,
|
||||
force_filetype=force_filetype,
|
||||
@@ -114,7 +126,7 @@ class TextoToKGCLI(object):
|
||||
else:
|
||||
job_id: str = self._client.create_unstructured_to_kg_job(
|
||||
input_text_path,
|
||||
ontology_json_path,
|
||||
ontology_json_path_final,
|
||||
ontology_owl_path,
|
||||
force_filetype=force_filetype,
|
||||
)
|
||||
@@ -211,13 +223,12 @@ class TextoToKGCLI(object):
|
||||
def main():
|
||||
"""Main entry point for the text-to-KG client console script."""
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("PYTHON_LOGGING_LEVEL", logging.INFO),
|
||||
stream=sys.stderr,
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
format=("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CleverSwarm unstructured text to Knowledge Graph triplets client for REST API."
|
||||
@@ -272,7 +283,7 @@ def main():
|
||||
parser.add_argument(
|
||||
"--ontology_json",
|
||||
type=str,
|
||||
help="Enriched ontology file with descriptions in JSON format",
|
||||
help="Optional enriched ontology file with descriptions in JSON format",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ontology_owl",
|
||||
@@ -306,10 +317,6 @@ def main():
|
||||
print("create-and-exit action requires --unstructured_text argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print("create-and-exit action requires --ontology_json argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print("create-and-exit action requires --ontology_owl argument.\n")
|
||||
parser.print_help()
|
||||
@@ -339,12 +346,6 @@ def main():
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_json argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_owl argument.\n"
|
||||
@@ -361,12 +362,6 @@ def main():
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_json argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_owl argument.\n"
|
||||
|
||||
@@ -595,7 +595,7 @@ class CleverSwarmClient(object):
|
||||
def create_unstructured_to_kg_job(
|
||||
self,
|
||||
unstructured_text_file: Path,
|
||||
ontology_file: Path,
|
||||
ontology_file: Path | None,
|
||||
ontology_spec_file: Path,
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
):
|
||||
@@ -604,7 +604,7 @@ class CleverSwarmClient(object):
|
||||
Login is requested if required by the server.
|
||||
If the upload is successful, it returns a job ID.
|
||||
:param unstructured_text_file: a valid file path containing unstructured text for conversion
|
||||
:param ontology_file: a valid file containing an ontology in JSON format.
|
||||
:param ontology_file: optional file containing an ontology in JSON format.
|
||||
:param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format.
|
||||
:param force_filetype: indicate the file type to force for the input file
|
||||
:return: the job ID where the files were placed.
|
||||
@@ -618,11 +618,6 @@ class CleverSwarmClient(object):
|
||||
f"{str(unstructured_text_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology files does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ontology_spec_file.exists() or not ontology_spec_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"OWL/XML ontology file does not exist or is not a file: "
|
||||
@@ -634,9 +629,11 @@ class CleverSwarmClient(object):
|
||||
params = {"file_type": force_filetype}
|
||||
files = [
|
||||
("unstructured", open(unstructured_text_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ontology_spec", open(ontology_spec_file, "rb")),
|
||||
]
|
||||
if ontology_file:
|
||||
files.append(("ontology", open(ontology_file, "rb")))
|
||||
|
||||
response = requests.post(
|
||||
self._base_url + "unstructured/with_ontology",
|
||||
headers=headers,
|
||||
@@ -669,7 +666,7 @@ class CleverSwarmClient(object):
|
||||
def create_unstructured_to_kg_wildcards_job(
|
||||
self,
|
||||
unstructured_text_file: Path,
|
||||
ontology_file: Path,
|
||||
ontology_file: Path | None,
|
||||
ontology_spec_file: Path,
|
||||
wildcards_query_file: Path,
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
@@ -679,7 +676,7 @@ class CleverSwarmClient(object):
|
||||
Login is requested if required by the server.
|
||||
If the upload is successful, it returns a job ID.
|
||||
:param unstructured_text_file: a valid file path containing unstructured text for conversion
|
||||
:param ontology_file: a valid file containing an ontology in JSON format.
|
||||
:param ontology_file: optional file containing an ontology in JSON format.
|
||||
:param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format.
|
||||
:param wildcards_query_file: a valid file path containing wildcard queries in JSON.
|
||||
:param force_filetype: indicate the file type to force for the input file
|
||||
@@ -694,11 +691,6 @@ class CleverSwarmClient(object):
|
||||
f"{str(unstructured_text_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology files does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ontology_spec_file.exists() or not ontology_spec_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"OWL/XML ontology file does not exist or is not a file: "
|
||||
@@ -716,10 +708,12 @@ class CleverSwarmClient(object):
|
||||
params = {"file_type": force_filetype}
|
||||
files = [
|
||||
("unstructured", open(unstructured_text_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ontology_spec", open(ontology_spec_file, "rb")),
|
||||
("wildcards", open(wildcards_query_file, "rb")),
|
||||
]
|
||||
if ontology_file:
|
||||
files.append(("ontology", open(ontology_file, "rb")))
|
||||
|
||||
response = requests.post(
|
||||
self._base_url + "unstructured/with_wildcards",
|
||||
headers=headers,
|
||||
|
||||
@@ -214,18 +214,75 @@ class TestTextoToKGCLIJobCreation:
|
||||
"test.owl"
|
||||
)
|
||||
|
||||
def test_create_job_invalid_json_file(self):
|
||||
"""Test job creation with invalid JSON file."""
|
||||
def test_create_job_with_invalid_json_file(self):
|
||||
"""Test job creation with invalid (non-existent) JSON file - should raise exception if JSON is provided."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file:
|
||||
text_file.write("Test text content")
|
||||
text_file.flush()
|
||||
|
||||
with pytest.raises(ClientException, match="Input ontology JSON file does not exist"):
|
||||
self.cli.create_job(
|
||||
text_file.name,
|
||||
"nonexistent.json",
|
||||
"test.owl"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file:
|
||||
owl_file.write('<rdf:RDF>test</rdf:RDF>')
|
||||
owl_file.flush()
|
||||
|
||||
# If ontology_json is provided but file doesn't exist, should raise exception
|
||||
with pytest.raises(ClientException, match="Specified optional Ontology JSON file does not exist"):
|
||||
self.cli.create_job(
|
||||
text_file.name,
|
||||
"nonexistent.json", # Non-existent JSON file - should raise exception
|
||||
owl_file.name
|
||||
)
|
||||
|
||||
def test_create_job_without_json_file(self):
|
||||
"""Test job creation without JSON file - should work since JSON is optional."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file:
|
||||
text_file.write("Test text content")
|
||||
text_file.flush()
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file:
|
||||
owl_file.write('<rdf:RDF>test</rdf:RDF>')
|
||||
owl_file.flush()
|
||||
|
||||
with patch.object(self.cli._client, 'create_unstructured_to_kg_job') as mock_create:
|
||||
mock_create.return_value = "test_job_123"
|
||||
|
||||
# Job creation should work without JSON file since it's optional
|
||||
result = self.cli.create_job(
|
||||
text_file.name,
|
||||
None, # No JSON file provided - should work
|
||||
owl_file.name
|
||||
)
|
||||
|
||||
assert result == "test_job_123"
|
||||
mock_create.assert_called_once()
|
||||
# Verify that None was passed for ontology_file since it wasn't provided
|
||||
call_args = mock_create.call_args
|
||||
assert call_args[0][1] is None # ontology_file parameter should be None
|
||||
|
||||
def test_create_job_with_empty_json_file_string(self):
|
||||
"""Test job creation with empty string for JSON file - should work since JSON is optional."""
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file:
|
||||
text_file.write("Test text content")
|
||||
text_file.flush()
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file:
|
||||
owl_file.write('<rdf:RDF>test</rdf:RDF>')
|
||||
owl_file.flush()
|
||||
|
||||
with patch.object(self.cli._client, 'create_unstructured_to_kg_job') as mock_create:
|
||||
mock_create.return_value = "test_job_123"
|
||||
|
||||
# Job creation should work with empty string for JSON file since it's optional
|
||||
result = self.cli.create_job(
|
||||
text_file.name,
|
||||
"", # Empty string for JSON file - should work
|
||||
owl_file.name
|
||||
)
|
||||
|
||||
assert result == "test_job_123"
|
||||
mock_create.assert_called_once()
|
||||
# Verify that None was passed for ontology_file since empty string is falsy
|
||||
call_args = mock_create.call_args
|
||||
assert call_args[0][1] is None # ontology_file parameter should be None
|
||||
|
||||
def test_create_job_invalid_owl_file(self):
|
||||
"""Test job creation with invalid OWL file."""
|
||||
|
||||
@@ -10,8 +10,11 @@ deps =
|
||||
pytest-html>=4.1.1
|
||||
requests>=2.32.4
|
||||
httmock>=1.3.0
|
||||
# Isolate coverage database per environment to prevent conflicts in parallel runs
|
||||
setenv =
|
||||
COVERAGE_FILE = {envdir}/.coverage.{envname}
|
||||
commands =
|
||||
pytest {posargs:tests} --html=unit_tests_report.html --self-contained-html --junitxml=unit_tests_report.xml --cov=cleverswarm_python_client
|
||||
pytest {posargs:tests} --html=unit_tests_report.html --self-contained-html --junitxml=unit_tests_report.xml --cov=src/cleverswarm_python_client --cov-config=.coveragerc
|
||||
|
||||
[testenv:lint]
|
||||
deps =
|
||||
@@ -36,8 +39,11 @@ deps =
|
||||
coverage>=5.5
|
||||
requests>=2.32.4
|
||||
httmock>=1.3.0
|
||||
# Use a dedicated coverage database for the coverage environment
|
||||
setenv =
|
||||
COVERAGE_FILE = {envdir}/.coverage
|
||||
commands =
|
||||
pytest {posargs:tests} --cov=cleverswarm_python_client --cov-report=term --cov-report=xml:coverage_report.xml --cov-report=html
|
||||
pytest {posargs:tests} --cov=src/cleverswarm_python_client --cov-config=.coveragerc --cov-report=term --cov-report=xml:coverage_report.xml --cov-report=html
|
||||
coverage report --fail-under=85
|
||||
|
||||
[testenv:format]
|
||||
|
||||
Reference in New Issue
Block a user