commit 1aebcfda0fcaea18b172a4d7a76a1807f4c8faf6 Author: CoreRasurae Date: Sat Sep 27 20:55:31 2025 +0100 feat: Initial commit of the SDK at version v0.0.1 diff --git a/.cz-config.js b/.cz-config.js new file mode 100644 index 0000000..5213949 --- /dev/null +++ b/.cz-config.js @@ -0,0 +1,48 @@ +'use strict'; + +module.exports = { + + types: [ + {value: 'feat', name: 'feat: A new feature'}, + {value: 'fix', name: 'fix: A bug fix'}, + {value: 'docs', name: 'docs: Documentation only changes'}, + {value: 'style', name: 'style: Changes that do not affect the meaning of the code\n (white-space, formatting, etc)'}, + {value: 'refactor', name: 'refactor: A code change that neither fixes a bug nor adds a feature'}, + {value: 'perf', name: 'perf: A code change that improves performance'}, + {value: 'test', name: 'test: Adding missing tests or correcting existing tests'}, + {value: 'build', name: 'build: Changes that affect the build system or external dependencies (example scopes: maven, gradle, npm, gulp)'}, + {value: 'ci', name: 'ci: Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs)'}, + {value: 'chore', name: 'chore: Other changes that dont modify src or test files'}, + {value: 'revert', name: 'revert: Reverts a previous commit'} + ], + + scopes: [ + {name: 'General'} + ], + + scopeOverrides: { + build: [ + {name: 'dependencies'}, + {name: 'versioning'}, + {name: 'release'}, + {name: 'build plugin'} + ], + ci: [ + {name: 'travis'} + ], + chore: [ + {name: 'commitizen'}, + {name: 'editorconfig'}, + {name: 'git'} + ], + docs: [ + {name: 'source'}, + {name: 'repo'}, + {name: 'build'} + ] + }, + + allowCustomScopes: true, + allowBreakingChanges: ['feat', 'fix', 'docs', 'style', 'refactor', 'perf', 'test', 'build', 'ci', 'chore', 'revert'] + +}; diff --git a/.cz.json b/.cz.json new file mode 100644 index 0000000..d4856e0 --- /dev/null +++ b/.cz.json @@ -0,0 +1 @@ +{ "path": "cz-customizable" } diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..27f6865 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +# top-most EditorConfig file +root = true + +[*] +#unix style line ending +end_of_line = lf +#newline at end of file +insert_final_newline = true +#charset UTF-8 +charset = utf-8 +#indent using spaces +indent_style = space +#4 spaces per indent +indent_size = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ecf07df --- /dev/null +++ b/.gitignore @@ -0,0 +1,63 @@ +# Useful examples +# https://github.com/github/gitignore + +# OS X +*.DS_Store + +# Java +*.jar +*.war +*.ear + +# C/C++ +*.so +*.dylib +*.dSYM +*.dll +*.jnilib + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# Directories +**/bin/ +**/classes/ +**/dist/ +**/include/ +**/nbproject/ +/.libs/ +/findbugs/ +/target/ + +#intellij +.idea/ +*.iml + +#logs +*.log + +#project specific +/src/genjava + +#Eclipse che +.che/ +.classpath +.project +.settings/ + +.tox/ +.aider* +.cache/ +.pki/ +**/__pycache__/ +knowledge_graph.ttl +.keras/ +build/ +**/*.egg-info/ +.local/ +.coverage* + +.aider* diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md new file mode 100644 index 0000000..3271095 --- /dev/null +++ b/ATTRIBUTIONS.md @@ -0,0 +1,9 @@ +This file contains all attributions and other notices that are required by law. This will contain all copyright +notices as well as any notes regarding licensing. + +## Copyright Notices + +The following are the list of all recognized copyright notices added by contributors to this project: + +Copyright (c) 2016 - present, Syncleus, Inc. All rights reserved. +Copyright (c) 2024 - present, CleverThis, Inc. All rights reserved. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8775031 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## v0.0.1 + * Initial release. + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f01ac44 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,72 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* Unwelcomed sexual attention or advances. +* Derogatory comments about a persons appearance, race, or sexual orientation. +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at +[abuse@cleverthis.com](mailto:abuse@cleverthis.com). All complaints will be +reviewed and investigated and will result in a response that is deemed necessary +and appropriate to the circumstances. The project team is obligated to maintain +confidentiality with regard to the reporter of an incident. Further details of +specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..12788ca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Contributing + +[![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/) +[![Semantic Versioning](https://img.shields.io/SemVer/2.0.0.png)](http://semver.org/spec/v2.0.0.html) +[![Matrix](https://img.shields.io/matrix/cleverthis%3Aqoto.org?server_fqdn=matrix.qoto.org&label=Matrix%20chat)](https://matrix.to/#/#CleverThis:qoto.org) + +When contributing to this repository, it is usually a good idea to first discuss the change you +wish to make via issue, email, or any other method with the owners of this repository before +making a change. This could potentially save a lot of wasted hours. + +Please note we have a code of conduct, please follow it in all your interactions with the project. + +## Development + +### Commit Message Format + +All commits on this repository must follow the +[Conventional Changelog standard](https://github.com/conventional-changelog/conventional-changelog-eslint/blob/master/convention.md). +It is a very simple format so you can still write commit messages by hand. However it is +highly recommended developers install [Commitizen](https://commitizen.github.io/cz-cli/), +it extends the git command and will make writing commit messages a breeze. All CleverThis +repositories are configured with local Commitizen configuration scripts. + +Getting Commitizen installed is usually trivial, just install it via npm. You will also +need to install the cz-customizable adapter which CleverThis repositories are configured +to use. + +```bash + +npm install -g commitizen@2.8.6 cz-customizable@4.0.0 +``` + +Below is an example of Commitizen in action. It replaces your usual `git commit` command +with `git cz` instead. The new command takes all the same arguments however it leads you +through an interactive process to generate the commit message. + +![Commitizen friendly](https://docs.cleverthis.com/public_media/commitizen.gif) + +Commit messages are used to automatically generate our changelogs, and to ensure +commits are searchable in a useful way. So please use the Commitizen tool and adhere to +the commit message standard or else we cannot accept Pull Requests without editing +them first. + +Below is an example of a properly formated commit message. + +``` +chore(Commitizen): Made repository Commitizen friendly. + +Added standard Commitizen configuration files to the repo along with all the custom rules. + +ISSUES CLOSED: #31 +``` + +### Pull Request Process + +1. Ensure that install or build dependencies do not appear in any commits in your code branch. +2. Ensure all commit messages follow the [Conventional Changelog](https://github.com/conventional-changelog/conventional-changelog-eslint/blob/master/convention.md) + standard explained earlier. +3. Update the CONTRIBUTORS.md file to add your name to it if it isn't already there (one entry + per person). +4. Adjust the project version to the new version that this Pull Request would represent. The + versioning scheme we use is [Semantic Versioning](http://semver.org/). +5. Your pull request will either be approved or feedback will be given on what needs to be + fixed to get approval. We usually review and comment on Pull Requests within 48 hours. diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 0000000..ab1ae6b --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,10 @@ +# Contributors + +* Luis Mendes + +# Details + +Below are some of the specific details of various contributions. + +* Luis Mendes has acted as Lead Developer, daily contributor, and Project Owner. +* This project was made possible thanks to considerable donation of time, money, and resources by CleverThis, Inc. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..376f52b --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE +include NOTICE +include README.md +recursive-include src *.py diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..6f2ec61 --- /dev/null +++ b/NOTICE @@ -0,0 +1,14 @@ +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..cafcf20 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# CleverSwarm Python client and SDK + +This repository provides CLI Python client applications for interacting with CleverSwarm micro-services. + +It also provides a SDK for easy integration of CleverSwarm with external applications (APPs) or frameworks. + +# Text to Knowledge-Graph extraction CLI APP + +This tool enables Knowledge-Graph (KG) extraction from unstructured text, provided an ontology. + +The ontology is actually split across two files OWL/RDF ontology file and a JSON ontology enriched with descriptions. +Wilcards triplets filtering is also supported, enabling KG extraction of the relevant triplets only. + +The extracted KG is exported to a file in the form of triplets in JSON-LD format. + +## Running the text to KG extraction CLI APP + +`cswarm_text_to_kg_client.py` is the end-user application for doing text extraction from the CLI. + +- Listing text to KG extraction jobs + + `cswarm_text_to_kg_client.py --action list-kg-jobs-and-exit --server_url http://127.0.0.1:8000/api/v0` + + will list all KG extraction jobs, but will exclude benchmark jobs from the listing. + + - Optionally the user may want to add `--username `, where `` denotes the actual username of the user on which behalf the extraction job will run. This will avoid the need to be prompted for the username. + + - The user may also specify `--token `, where `` is a valid token for the user. That way no manual login will be required. + + - The user may also indicate the option `--detailed` to get more detailed information. + + +- Doing an unstructured text to KG extraction + + `cswarm_text_to_kg_client.py --input_prefix unstructured_tests --unstructured_text "text 1.txt" --ontology_json ontology.json --ontology_owl ontology.owl --server_url http://localhost/api/v0` + + will create a regular text to KG extraction job based on the input files inside the `unstructured_tests` folder as indicated by `--input-prefix`. + Secondly the application will wait for KG extraction job completion, and then it will download the results file. + The output folder will be the current folder, since option `--output_prefix` is not provided. Finally the extraction job will be deleted from the REST API server. + + - The additional optional arguments described for the jobs listing action are also valid for this default action. + +- Doing an unstructured text to KG extraction with wildcards filtering + + `cswarm_text_to_kg_client.py --input_prefix wildxard_tests --unstructured_text text.txt --ontology_json ontology.json --ontology_owl ontology.owl --wildcards ~/Documents/wildcard_query_ist.json --server_url http://localhost/api/v0` + + the additional `--wildcards` option is what differentiates from the regular text to KG extraction to the wildcards text to KG extraction. + Please note that `--wildcards` does not inherit the input folder prefix, so a full path to the wildcards file should be specified. + + - The additional optional arguments described for the jobs listing action are also valid for this default action. + +- For additional command line arguments, please do: + + `cswarm_text_to_kg_client.py --help` + +# Benchmarking CleverSwarm + +Make sure to install CleverSwarm Python client: `pip install --index-url https://git.cleverthis.com/api/packages/cleverlibre/pypi/simple/ cleverswarm-python-client` and to clone the CleverSwarm Benchmark repo: `git clone ssh://git@git.cleverthis.com/cleverswarm/cleverswarm-benchmark.git` + +## Running the benchmark +- Running the full set of original tests with no previous tests in the local filesystem cache: + + `cswarm_benchmark_client --ontologies_ids 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 15 17 18 19` + + **NOTE 1**: This assumes the default benchmark datasets prefix of './data/dbpedia'. If another prefix is required please provide the option `--bench_data_prefix ` + + **NOTE 2**: Re-running the command will take the same time to execute and will overwrite all previously locally cached results for the specified tests cases. + +- Running the full set of original tests with some previous tests in the local filesystem cache to obtain the average of all the tests: + + For this case, lets assume that the first request benchmarked tests were 1, 2, 3, 4, 5 and 6, so that a previous benchmark job ran like so: + + `cswarm_benchmark_client --ontologies_ids 1 2 3 4 5 6` + + So, all is needed is to specify that the new server benchmark job can exclude the jobs from 1 to 6 and just start at 7, for that we can utilize the option: + + `--ontologies_ids 7 8 9 10 11 12 13 14 15 15 17 18 19` + + however, since we want want the average scores for the jobs 1-19, we also need to append this option: + + `--local_metrics_ontologies_ids 1 2 3 4 5 6` + + the final command line, will then look like this: + + `cswarm_benchmark_client --ontologies_ids 7 8 9 10 11 12 13 14 15 15 17 18 19 --local_metrics_ontologies_ids 1 2 3 4 5 6` + +- Running the evalution metrics for cached test cases, supposing that all the test cases were already processed on the server and their results downloaded to the local filesystem, then we can do: + + `cswarm_benchmark_client.py --action metrics-and-exit --local_metrics_ontologies_ids 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 15 17 18 19` + + +## Other details about the Benchmark client tool +- For general help do: + + `cswarm_benchmark_client.py -h` + +- To specify a CleverThis CleverSwarm server URL do: + + `cswarm_benchmark_client.py --server_url https://cleverthis.com` + +- To specify a username for authentication with the server do: + + `cswarm_benchmark_client.py --username ` + +- To specify a given action for the benchmark client, do: + + `cswarm_benchmark_client.py --action ` + + +## Detail of the available Benchmark client tool actions +- `--action create-and-exit` creates a benchmark job in the server for the specified test cases and exits. + +- `--action delete-and-exit` deletes a benchmark job in the server and exits. + +- `--action metrics-and-exit` computes the summarized evaluation metrics/scores for the specified local filesystem cached test cases and exits. + +- `--action metrics-and-exit --detailed` computes the detailed evaluation metrics/scores for the specified local filesystem cached test cases and exits. + +- `--action download-metrics-and-exit` downloads the test cases results from the server to the local filesystem cache. + +- `--action list-benchmark-jobs-and-exit` to obatin a summarized list of the benchmark jobs in the server, followed by exit. + +- `--action list-benchmark-jobs-and-exit --detailed` to obatin a detailed list of the benchmark jobs in the server, followed by exit. + +- `--action list-ontologies-ids-and-exit` lists the known ontologies test cases ids and names. + +- `--action create-download-metrics-delete` performs the full client benchmark process, composed of server jobs creation, polling of the job completion, downloading the jobs results to the local filesystem benchmark results folder, copmuting evaluation metrics locally in the local machine and deleting the benchmark job in the server. **NOTE** This is the default, if no action is specified in the command-line. + +- `--action create-download-metrics` similar to the above, but does not delete the benchmark job in the server at the end of the processing. + + +## Reusing the client code library + +- Please look into `client/cleverswarm_client.py` and associated modules. \ No newline at end of file diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..e2b453e --- /dev/null +++ b/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +# Add the src directory to the Python path +src_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "src")) +if src_path not in sys.path: + sys.path.insert(0, src_path) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f1ad25c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,17 @@ +[tool:pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = + --verbose + --tb=short + --cov=src/cleverswarm_python_client + --cov-report=html + --cov-report=term-missing + --cov-fail-under=85 + --cov-branch +markers = + unit: Unit tests + integration: Integration tests + slow: Slow running tests diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..61ff295 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,13 @@ +-r requirements.txt +pytest>=7.0.0 +pytest-cov>=4.1.0 +pytest-html>=3.2.0 +coverage==7.8.2 +pytest-cov>=4.1.0 +httmock>=1.3.0 +black>=25.9.0 +isort>=5.12.0 +flake8>=6.0.0 +mypy>=1.3.0 +sphinx>=7.0.0 +sphinx-rtd-theme>=1.2.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..05a991d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requests>=2.32.4 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..91ab48d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,73 @@ +[metadata] +name = cleverswarm_python_client +version = 0.0.1 +description = A set of Python client utilities and SDK for interacting with CleverSwarm micro-services +long_description = file: README.md +long_description_content_type = text/markdown +author = CleverThis +author_email = luis.mendes@cleverthis.com +license = Apache +license_files = LICENSE +url = https://git.cleverthis.com/cleverlibre/cleverswarm-python-client +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: Unix + Operating System :: POSIX + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 +keywords = + Knowledge Graph + Text Extraction + CleverSwarm SDK + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.8 +install_requires = + requests>=2.32.3 + +[options.packages.find] +where = src +include = cleverswarm_python_client* + +[options.package_data] +* = *.txt, *.rst, *.md +cleverswarm_python_client = data/*.dat, *.json + +[options.entry_points] +console_scripts = + cswarm-benchmark = cleverswarm_python_client.cswarm_benchmark_client:main + cswarm-text-to-kg = cleverswarm_python_client.cswarm_text_to_kg_client:main + +[options.extras_require] +dev = + pytest>=6.0.0 + pytest-html>=3.2.0 + pytest-cov>=2.12.0 + moto>=2.0.0 + black>=21.5b2 + isort>=5.9.1 + flake8>=3.9.2 + mypy>=0.812 + +[flake8] +max-line-length = 120 +exclude = .git,__pycache__,build,dist + +[isort] +profile = black +line_length = 120 + +[mypy] +python_version = 3.8 +warn_return_any = True +warn_unused_configs = True +disallow_untyped_defs = True +disallow_incomplete_defs = True diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..118a8da --- /dev/null +++ b/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup, find_namespace_packages + +setup( + packages=find_namespace_packages(where="src"), + package_dir={"": "src"}, + include_package_data=True, +) diff --git a/src/cleverswarm_python_client/__init__.py b/src/cleverswarm_python_client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cleverswarm_python_client/cswarm_benchmark_client.py b/src/cleverswarm_python_client/cswarm_benchmark_client.py new file mode 100644 index 0000000..4e1e668 --- /dev/null +++ b/src/cleverswarm_python_client/cswarm_benchmark_client.py @@ -0,0 +1,496 @@ +""" +CleverSwarm - text to knowledge graph extraction benchmark client. + + Copyright (c) 2016 - present Syncleus, Inc. + Copyright (c) 2016 - present CleverThis, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import argparse +import logging +from pathlib import Path +from typing import Dict, List, Set + +from cleverswarm_python_client.libs.cleverswarm_client import CleverSwarmClient +from cleverswarm_python_client.libs.evaluation.detailed_eval import EvaluateResults +from cleverswarm_python_client.libs.exceptions import ( + ClientException, + JobNotFoundException, +) +from cleverswarm_python_client.libs.job_enums import JobStatus + +logger = logging.getLogger("Benchmark-client main") + + +class BenchmarkCLI(object): + def __init__( + self, + base_path: str, + base_url: str = "http://localhost:8000/api/v0/", + detailed_metrics: bool = False, + username: str = None, + token: str = None, + ): + self._all_ontologies = { + 1: "1_university", + 2: "2_musicalwork", + 3: "3_airport", + 4: "4_building", + 5: "5_athlete", + 6: "6_politician", + 7: "7_company", + 8: "8_celestialbody", + 9: "9_astronaut", + 10: "10_comicscharacter", + 11: "11_meanoftransportation", + 12: "12_monument", + 13: "13_food", + 14: "14_writtenwork", + 15: "15_sportsteam", + 16: "16_city", + 17: "17_artist", + 18: "18_scientist", + 19: "19_film", + 20: "20_simple", + 21: "21_complex", + } + + self._base_url = base_url + self._base_path: Path = Path(base_path).resolve() + self._benchmark_path = "benchmark_data" + self._benchmark_output = self._benchmark_path + "/local_metrics" + self._input_template = "test_new/ont_$$onto$$_test.jsonl" + self._onto_template = ( + "ontologies_enriched/$$onto$$_ontology_with_descriptions.json" + ) + self._ground_truth_template = "ground_truth_new/ont_$$onto$$_ground_truth.jsonl" + self._output_template = ( + self._benchmark_path + "/outputs_new/$$onto$$_output.jsonl" + ) + self._detailed_metrics = detailed_metrics + + self._client: CleverSwarmClient = CleverSwarmClient( + self._base_url, username=username, token=token + ) + + def find_invalid_ontologies(self, ontologies_ids: List[int]) -> List[int]: + """ + Find ontologies IDs that are unknown to the benchmark dataset. + :param ontologies_ids: the ids to be verified + :return: the list of invalid ids, or empty if no invalid + """ + invalids: List[int] = [] + for ontology_id in ontologies_ids: + if ontology_id not in self._all_ontologies.keys(): + invalids.append(ontology_id) + return invalids + + def create_benchmark(self, ontologies_ids: List[int]) -> str: + """ + Creates a new benchmark job in the CleverThis REST API server. + :param ontologies_ids: the ontologies ids of the ontologies that are to be benchmarked. + :return: the job_id of the created job, or None if no job was created + """ + if len(ontologies_ids) == 0: + return None + + ontologies = [ + self._all_ontologies[ontology_id] for ontology_id in ontologies_ids + ] + for idx in range(len(ontologies)): + ontology_name: str = ontologies[idx] + test_file: Path = self._base_path / self._input_template.replace( + "$$onto$$", ontology_name + ) + ontology_file: Path = self._base_path / self._onto_template.replace( + "$$onto$$", ontology_name + ) + ground_truth_file: Path = ( + self._base_path + / self._ground_truth_template.replace("$$onto$$", ontology_name) + ) + last_fileset = idx == len(ontologies) - 1 + if idx == 0: + job_id = self._client.create_benchmark_job( + test_file, ontology_file, ground_truth_file, last_fileset + ) + else: + self._client.append_to_benchmark_job( + job_id, test_file, ontology_file, ground_truth_file, last_fileset + ) + + return job_id + + def poll_job_to_completion(self, job_id: str) -> None: + """ + Polls a job by its job id and waits for the job completion in the server. + It throws an exception if the job failed or is in a state that cannot be executed. + :param job_id: the job id of the job to be executed. + """ + status: JobStatus = self._client.get_job_status(job_id) + if status == JobStatus.Failed: + raise ClientException(f"Job with ID: {job_id} finished without completing.") + if status == JobStatus.Completed: + return + if status != JobStatus.ReadyForProcessing and status != JobStatus.Processing: + raise ClientException( + f"Job with ID: {job_id} is not processing, nor is it ready for processing." + ) + final_status: JobStatus = self._client.poll_job_ready_or_failed(job_id) + if final_status != JobStatus.Completed: + raise ClientException(f"Job with ID: {job_id} finished without completing.") + + def download_job_results(self, job_id: str, ontologies_ids: List[int]) -> None: + """ + Download the results from the server, for the specified job and the specified ontologies_ids. + :param job_id: the job_id from which the results are to be retrieved + :param ontologies_ids: the ontologies_ids that are to be retrieved + """ + ontologies = [ + self._all_ontologies[ontology_id] for ontology_id in ontologies_ids + ] + + output_filenames = [] + for idx in range(len(ontologies)): + ontology_name: str = ontologies[idx] + output_file: Path = self._base_path / self._output_template.replace( + "$$onto$$", ontology_name + ) + output_filenames.append(output_file) + + self._client.retrieve_benchmark_files(job_id, output_filenames) + print( + f"Benchmark job with ID: {job_id}, produced the following output files: {output_filenames}" + ) + + def print_ontologies_ids(self) -> None: + """ + Prints the list of all known benchmark ontologies and their IDs. + """ + print("ID, Name") + for item in self._all_ontologies.items(): + print("{:>2}, {}".format(item[0], item[1])) + + def list_server_jobs(self) -> Dict: + """ + List all benchmark jobs in the server that are owned by the current logged in user. + :return: JSON dictionary with job listing and respective states. + """ + json_data = self._client.get_jobs_list() + return json_data + + def extract_server_job_ontologies(self, job_id: str) -> List[int]: + """ + Obtains the ontologies ids from a given server job, identified by its ID. + :param job_id: the job id of the server job to be consulted. + :return: the ontologies ids found in the specified benchmark server job + """ + found_job = self._client.get_job_details(job_id) + + if found_job is None: + raise JobNotFoundException( + f"Could not find job with ID: {job_id} in the server" + ) + + results: List[int] = [] + for source in found_job["ontology_sources"]: + idx_str = source.split("_")[0] + results.append(int(idx_str)) + + return results + + def print_server_jobs(self, detailed: bool = False) -> None: + """ + Prints a summarized or detailed status of the benchmark jobs in the server. + """ + jobs = self.list_server_jobs() + for job in jobs: + print("Job ID: {}".format(job["id"])) + print("Created at: {}".format(job["created"])) + print("Type: {}".format(job["type"])) + print("Status: {}".format(job["status"])) + print("Retries: {}".format(job["retries_count"])) + print("Input file type: {}".format((job["file_type"]))) + if detailed: + print("Unstructured text input files:") + for source in job["unstructured_sources"]: + print(f" {source}") + print("Ontology input files:") + for source in job["ontology_sources"]: + print(f" {source}") + print("Ground-truth input files:") + for source in job["ground_truth_sources"]: + print(f" {source}") + print("----------------------------") + + def delete_server_job(self, job_id: str) -> None: + """ + Deletes a given job by its ID. A job can only be deleted if it is in a state that allows it. + :param job_id: the server job id + """ + del_status = self._client.delete_job(job_id) + if not del_status: + print("Failed to delete Job") + else: + print("Job was successfully deleted") + + def evaluate_results_locally(self, ontologies_ids: List[int]) -> Path: + """ + Evaluates the extraction data obtained from the server and in the local machine to obtain the benchmark data. + The scores can be either detailed or summarized. + :param ontologies_ids: the ids of the ontologies to be benchmarked, it can include already downloaded ids + from previous benchmark jobs, for which the data has already been downloaded. + """ + ontologies = [ + self._all_ontologies[ontology_id] for ontology_id in ontologies_ids + ] + json_dict: Dict = { + "onto_list": ontologies, + "path_patterns": { + "sys": str(self._base_path / self._output_template), + "gt": str(self._base_path / self._ground_truth_template), + "onto": str(self._base_path / self._onto_template), + "output": str( + self._base_path + / self._benchmark_output + / "ont_$$onto$$_eval_results.jsonl" + ), + }, + "overall_output": str( + self._base_path / self._benchmark_output / "avg_eval_results.jsonl" + ), + } + + EvaluateResults(json_dict, self._detailed_metrics) + return self._base_path / self._benchmark_output + + +def main(): + """Main entry point for the benchmark client console script.""" + parser = argparse.ArgumentParser( + description="CleverSwarm benchmark client for REST API." + ) + parser.add_argument( + "--ontologies_ids", + nargs="+", + type=int, + help="List of ontologies IDs to be processed", + ) + parser.add_argument( + "--local_metrics_ontologies_ids", + nargs="+", + type=int, + help="List of ontologies IDs to be benchmarked (can include locally cached test ontologies results).", + ) + parser.add_argument( + "--bench_data_prefix", + type=str, + default="./data/dbpedia", + help="Path prefix to reach the common benchmark folder structure.", + ) + parser.add_argument( + "--username", type=str, help="Username for logging in into server" + ) + parser.add_argument( + "--token", + type=str, + help="Valid Bearer Token value to avoid the need for authentication", + ) + parser.add_argument( + "--server_url", + type=str, + default="http://localhost:8000/api/v0/", + help="REST server URL", + ) + parser.add_argument( + "--action", + choices=[ + "create-and-exit", + "delete-and-exit", + "metrics-and-exit", + "download-metrics-and-exit", + "list-benchmark-jobs-and-exit", + "list-ontologies-ids-and-exit", + "create-download-metrics-delete", + "create-download-metrics", + ], + type=str, + help="Define action to be performed", + default="create-download-metrics-delete", + ) + parser.add_argument( + "--job_id", + type=str, + help="Identify a specific job. To be used with create-and-exit, delete-and-exit, download-metrics-and-exit", + ) + parser.add_argument( + "--detailed", + default=False, + action="store_true", + ) + + args = parser.parse_args() + + create: bool = False + poll: bool = False + download: bool = False + metrics: bool = False + delete: bool = False + list_jobs: bool = False + list_ontologies: bool = False + + if args.action == "create-and-exit": + create = True + poll = False + if args.ontologies_ids is None: + print("Create-and-exit action requires --ontologies_ids argument.\n") + exit(-1) + elif args.action == "delete-and-exit": + delete = True + if args.job_id is None: + parser.print_help() + exit(-1) + elif args.action == "metrics-and-exit": + metrics = True + if args.local_metrics_ontologies_ids is None: + print("Metrics-and-exit action requires --ontologies_ids argument.\n") + parser.print_help() + exit(-1) + elif args.action == "download-metrics-and-exit": + poll = True + download = True + metrics = True + if args.job_id is None: + print("Download-metrics-and-exit action requires --job_id argument.\n") + parser.print_help() + exit(-1) + elif args.action == "list-benchmark-jobs-and-exit": + list_jobs = True + elif args.action == "create-download-metrics-delete": + create = True + poll = True + download = True + metrics = True + delete = True + if args.ontologies_ids is None: + print( + "Create-download-metrics-delete action requires --ontologies_ids argument.\n" + ) + parser.print_help() + exit(-1) + elif args.action == "create-download-metrics": + create = True + poll = True + download = True + metrics = True + if args.ontologies_ids is None: + print( + "Create-download-metrics action requires --ontologies_ids argument.\n" + ) + parser.print_help() + exit(-1) + elif args.action == "list-ontologies-ids-and-exit": + list_ontologies = True + + url: str = args.server_url + if not url.endswith("/"): + url += "/" + + ontologies_ids: List[int] = None + if args.ontologies_ids is not None: + ontologies_ids_set: Set[int] = set(args.ontologies_ids) + ontologies_ids: List[int] = sorted(list(ontologies_ids_set)) + + cli: BenchmarkCLI = BenchmarkCLI( + args.bench_data_prefix, + url, + args.detailed, + username=args.username, + token=args.token, + ) + try: + job_id = args.job_id + + if ontologies_ids is not None: + invalids = cli.find_invalid_ontologies(ontologies_ids) + if len(invalids) > 0: + print("Invalid IDs found in option --ontologies_ids.") + print(f"The following ontologies ids are invalid: {invalids}\n") + print( + "Use --list-ontologies-ids-and-exit to find the supported benchmark ontologies ids.\n" + ) + print("Type 'python main.py -h' for Help") + exit(-1) + + if args.local_metrics_ontologies_ids is not None: + invalids = cli.find_invalid_ontologies(args.local_metrics_ontologies_ids) + if len(invalids) > 0: + print("Invalid IDs found in option --local_metrics_ontologies_ids.") + print(f"The following ontologies ids are invalid: {invalids}\n") + print( + "Use --list-ontologies-ids-and-exit to find the supported benchmark ontologies ids.\n" + ) + print("Type 'python main.py -h' for Help") + exit(-1) + + if list_ontologies: + cli.print_ontologies_ids() + + if create: + job_id: str = cli.create_benchmark(ontologies_ids) + print(f"Successfully created benchmark job with ID: {job_id}.") + + if poll: + cli.poll_job_to_completion(job_id) + + if download: + if not create or ontologies_ids is None: + ontologies_ids = cli.extract_server_job_ontologies(job_id) + cli.download_job_results(job_id, ontologies_ids) + + if metrics: + local_ontologies_ids: Set = set() + if ontologies_ids is not None and create: + local_ontologies_ids.update(ontologies_ids) + elif job_id is not None: + print("Checking ontologies present in Job ID: {job_id}.") + local_ontologies_ids.update(cli.extract_server_job_ontologies(job_id)) + if args.local_metrics_ontologies_ids is not None: + local_ontologies_ids.update(args.local_metrics_ontologies_ids) + + local_ontologies_ids: List[int] = sorted(list(local_ontologies_ids)) + print( + "Considering ontologies IDs: {} for local metrics".format( + local_ontologies_ids + ) + ) + output_folder_path = cli.evaluate_results_locally(local_ontologies_ids) + print(f"Results scores written to: {str(output_folder_path)}") + + if delete: + cli.delete_server_job(job_id) + + if list_jobs: + cli.print_server_jobs(args.detailed) + except ClientException as error: + print(str(error)) + exit(-1) + except Exception as error: + logger.error(f"Unexpected error occurred: {error}", exc_info=error) + print("Unexpected error occurred, check the logs for more details") + exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/cleverswarm_python_client/cswarm_text_to_kg_client.py b/src/cleverswarm_python_client/cswarm_text_to_kg_client.py new file mode 100644 index 0000000..1a4bc9b --- /dev/null +++ b/src/cleverswarm_python_client/cswarm_text_to_kg_client.py @@ -0,0 +1,409 @@ +""" +CleverSwarm - unstructured text to knowledge graph triplets extractor client. + + Copyright (c) 2016 - present Syncleus, Inc. + Copyright (c) 2016 - present CleverThis, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import argparse +import logging +from pathlib import Path +from typing import Any, Dict + +from cleverswarm_python_client.libs.cleverswarm_client import CleverSwarmClient +from cleverswarm_python_client.libs.exceptions import ClientException +from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI +from cleverswarm_python_client.libs.job_enums import JobStatus, JobType + +logger = logging.getLogger("Text-to-KG-client main") + + +class TextoToKGCLI(object): + def __init__( + self, + base_url: str, + detailed: bool = False, + input_prefix: str = None, + output_prefix: str = None, + username: str = None, + token: str = None, + ): + + self._input_prefix: Path = Path(input_prefix) if input_prefix else Path("../..") + self._output_prefix: Path = ( + Path(output_prefix) if output_prefix else Path("../..") + ) + self._username: str = username + + self._input_prefix = self._input_prefix.resolve() + self._output_prefix = self._output_prefix.resolve() + + if not self._input_prefix.exists() or not self._input_prefix.is_dir(): + raise ClientException( + f"Input prefix does not exist or is not a folder: {input_prefix}" + ) + + self._output_prefix.mkdir(parents=True, exist_ok=True) + + if not self._output_prefix.exists() or not self._output_prefix.is_dir(): + raise ClientException( + f"Input prefix could not be created or is not a folder: {output_prefix}" + ) + + self._client: CleverSwarmClient = CleverSwarmClient(base_url, token, username) + + self._detailed = detailed + + def create_job( + self, + input_text: str, + ontology_json: str, + ontology_owl: str, + wildcards: str = None, + force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect, + ): + input_text_path: Path = self._input_prefix / input_text + ontology_json_path: Path = self._input_prefix / ontology_json + ontology_owl_path: Path = self._input_prefix / ontology_owl + wildcards_path: Path = Path(wildcards) if wildcards else None + + if not input_text_path.exists() or not input_text_path.is_file(): + raise ClientException( + f"Unstructured input text does not exist or is not a file: {str(input_text_path)}" + ) + + if not ontology_json_path.exists() or not ontology_json_path.is_file(): + raise ClientException( + f"Input ontology JSON file does not exist or is not a file: {str(ontology_json_path)}" + ) + + if not ontology_owl_path.exists() or not ontology_owl_path.is_file(): + raise ClientException( + f"Input ontology OWL file does not exist or is not a file: {str(ontology_owl_path)}" + ) + + wildcards_job: bool = False + if wildcards_path is not None: + wildcards_job = True + if not wildcards_path.exists() or not wildcards_path.is_file(): + raise ClientException( + f"Input wildcards file does not exist or is not a file: {str(wildcards_path)}" + ) + + if wildcards_job: + job_id: str = self._client.create_unstructured_to_kg_wildcards_job( + input_text_path, + ontology_json_path, + ontology_owl_path, + wildcards_path, + force_filetype=force_filetype, + ) + else: + job_id: str = self._client.create_unstructured_to_kg_job( + input_text_path, + ontology_json_path, + ontology_owl_path, + force_filetype=force_filetype, + ) + + return job_id + + def poll_job_to_completion(self, job_id: str) -> None: + """ + Polls a job by its job id and waits for the job completion in the server. + It throws an exception if the job failed or is in a state that cannot be executed. + :param job_id: the job id of the job to be executed. + """ + status: JobStatus = self._client.get_job_status(job_id) + if status == JobStatus.Failed: + raise ClientException(f"Job with ID: {job_id} finished without completing.") + if status == JobStatus.Completed: + return + if status != JobStatus.ReadyForProcessing and status != JobStatus.Processing: + raise ClientException( + f"Job with ID: {job_id} is not processing, nor is it ready for processing." + ) + final_status: JobStatus = self._client.poll_job_ready_or_failed(job_id) + if final_status != JobStatus.Completed: + raise ClientException(f"Job with ID: {job_id} finished without completing.") + + def download_job_result(self, job_id: str) -> None: + details: Dict[str, Any] = self._client.get_job_details(job_id) + + if len(details["unstructured_sources"]) == 0: + raise ClientException( + f"Unstructured to KG job: {job_id} is not valid. It has no input files." + ) + if len(details["unstructured_sources"]) > 1: + raise ClientException( + f"Unstructured to KG job: {job_id} is not valid. It has more than one input files." + ) + input_filename: str = details["unstructured_sources"][0] + output_filename: str = input_filename + ".json" + output_filepath: Path = self._output_prefix / output_filename + self._client.retrieve_unstructured_to_kg_files(job_id, output_filepath) + if self._detailed: + print("Job result was downloaded to: {}".format(str(output_filepath))) + + def delete_server_job(self, job_id: str) -> None: + """ + Deletes a given job by its ID. A job can only be deleted if it is in a state that allows it. + :param job_id: the server job id + """ + del_status = self._client.delete_job(job_id) + if not del_status: + print("Failed to delete Job") + else: + print("Job was successfully deleted") + + def list_server_jobs(self) -> Dict: + """ + List all text to KG extraction jobs in the server that are owned by the current logged-in user. + :return: JSON dictionary with job listing and respective states. + """ + json_data = self._client.get_jobs_list(is_benchmark=False) + return json_data + + def print_server_jobs(self, is_detailed: bool = False) -> None: + """ + Prints a summarized or detailed status of the text to KG extraction jobs in the server. + """ + jobs = self.list_server_jobs() + for job in jobs: + print("Job ID: {}".format(job["id"])) + print("Created at: {}".format(job["created"])) + print("Type: {}".format(job["type"])) + print("Status: {}".format(job["status"])) + print("Retries: {}".format(job["retries_count"])) + print("Input file type: {}".format((job["file_type"]))) + if is_detailed: + print("Unstructured text input files:") + for source in job["unstructured_sources"]: + print(f" {source}") + print("Ontology JSON input files:") + for source in job["ontology_sources"]: + print(f" {source}") + print("Ontology OWL/XML input files:") + for source in job["ontology_spec_sources"]: + print(f" {source}") + if JobType(job["type"]) == JobType.UnstructuredWithWildcards: + print("Wildcards input files:") + for source in job["wildcards_sources"]: + print(f" {source}") + print("----------------------------") + + +def main(): + """Main entry point for the text-to-KG client console script.""" + parser = argparse.ArgumentParser( + description="CleverSwarm unstructured text to Knowledge Graph triplets client for REST API." + ) + parser.add_argument( + "--input_prefix", + type=str, + default=".", + help="Path prefix for the common input folder structure.", + ) + parser.add_argument( + "--output_prefix", type=str, default=".", help="Path to the output folder." + ) + parser.add_argument( + "--username", type=str, help="Username for logging in into server" + ) + parser.add_argument( + "--token", + type=str, + help="Valid Bearer Token value to avoid the need for authentication", + ) + parser.add_argument( + "--server_url", + type=str, + default="http://localhost:8000/api/v0/", + help="REST server URL", + ) + parser.add_argument( + "--action", + choices=[ + "create-and-exit", + "delete-and-exit", + "download-kg-and-exit", + "list-kg-jobs-and-exit", + "create-download-kg-delete", + "create-download-kg", + ], + type=str, + help="Define action to be performed", + default="create-download-kg-delete", + ) + parser.add_argument( + "--job_id", + type=str, + help="Identify a specific job. To be used with create-and-exit, delete-and-exit, download-metrics-and-exit", + ) + parser.add_argument( + "--unstructured_text", + type=str, + help="Unstructured text input file for the Knowledge Graph triplet extraction", + ) + parser.add_argument( + "--ontology_json", + type=str, + help="Enriched ontology file with descriptions in JSON format", + ) + parser.add_argument( + "--ontology_owl", + type=str, + help="OWL ontology file in XML format", + ) + parser.add_argument( + "--wildcards", + type=str, + help="Wildcards ontology file. If specified, a wildcards filtering job will be created instead " + "of a regular KG extraction.", + ) + parser.add_argument( + "--detailed", + default=False, + action="store_true", + ) + + args = parser.parse_args() + + create = False + poll = False + download = False + delete = False + list_jobs = False + + if args.action == "create-and-exit": + create = True + poll = False + if args.unstructured_text is None: + print("create-and-exit action requires --unstructured_text argument.\n") + parser.print_help() + exit(-1) + if args.ontology_json is None: + print("create-and-exit action requires --ontology_json argument.\n") + parser.print_help() + exit(-1) + if args.ontology_owl is None: + print("create-and-exit action requires --ontology_owl argument.\n") + parser.print_help() + exit(-1) + elif args.action == "delete-and-exit": + delete = True + if args.job_id is None: + parser.print_help() + exit(-1) + elif args.action == "download-kg-and-exit": + poll = True + download = True + if args.job_id is None: + print("download-kg-and-exit action requires --job_id argument.\n") + parser.print_help() + exit(-1) + elif args.action == "list-kg-jobs-and-exit": + list_jobs = True + elif args.action == "create-download-kg-delete": + create = True + poll = True + download = True + delete = True + if args.unstructured_text is None: + print( + "create-download-kg-delete action requires --unstructured_text argument.\n" + ) + parser.print_help() + exit(-1) + if args.ontology_json is None: + print( + "create-download-kg-delete action requires --ontology_json argument.\n" + ) + parser.print_help() + exit(-1) + if args.ontology_owl is None: + print( + "create-download-kg-delete action requires --ontology_owl argument.\n" + ) + parser.print_help() + exit(-1) + elif args.action == "create-download-kg": + create = True + poll = True + download = True + if args.unstructured_text is None: + print( + "create-download-kg-delete action requires --unstructured_text argument.\n" + ) + parser.print_help() + exit(-1) + if args.ontology_json is None: + print( + "create-download-kg-delete action requires --ontology_json argument.\n" + ) + parser.print_help() + exit(-1) + if args.ontology_owl is None: + print( + "create-download-kg-delete action requires --ontology_owl argument.\n" + ) + parser.print_help() + exit(-1) + + url: str = args.server_url + if not url.endswith("/"): + url += "/" + + try: + job_id = args.job_id + cli = TextoToKGCLI( + base_url=url, + detailed=args.detailed, + input_prefix=args.input_prefix, + output_prefix=args.output_prefix, + username=args.username, + token=args.token, + ) + if create: + job_id = cli.create_job( + args.unstructured_text, + args.ontology_json, + args.ontology_owl, + args.wildcards, + ) + + if poll: + cli.poll_job_to_completion(job_id) + + if download: + cli.download_job_result(job_id) + + if delete: + cli.delete_server_job(job_id) + + if list_jobs: + cli.print_server_jobs(args.detailed) + + except ClientException as msg: + print(str(msg)) + exit(-1) + except Exception as error: + logger.error(f"Unexpected error occurred: {error}", exc_info=error) + print("Unexpected error occurred, check the logs for more details") + exit(-1) + + +if __name__ == "__main__": + main() diff --git a/src/cleverswarm_python_client/libs/__init__.py b/src/cleverswarm_python_client/libs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/cleverswarm_python_client/libs/cleverswarm_client.py b/src/cleverswarm_python_client/libs/cleverswarm_client.py new file mode 100644 index 0000000..e874dfa --- /dev/null +++ b/src/cleverswarm_python_client/libs/cleverswarm_client.py @@ -0,0 +1,817 @@ +""" +Module for interfacing with CleverSwarm REST API server + + Copyright (c) 2016 - present Syncleus, Inc. + Copyright (c) 2016 - present CleverThis, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +import logging +import os +import time +from getpass import getpass +from pathlib import Path +from typing import Any, Dict, List, Self + +import requests + +from cleverswarm_python_client.libs.exceptions import ( + BaseUrlMissingException, + ClientException, + ConnectionErrorException, + ConnectionTimeoutException, + InvalidFileException, + InvalidUsernameOrPasswordException, + JobNotFoundException, + RequestErrorException, + UnauthorizedException, + UnexpectedConditionException, +) +from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI +from cleverswarm_python_client.libs.job_enums import JobStatus, JobType +from cleverswarm_python_client.libs.response_type_enum import ResponseTypeAPI + +logger = logging.getLogger("Benchmark-client") + + +class CleverSwarmClient(object): + """ + CleverSwarmClient is a class that provides an abstraction for interacting with a CleverSwarm REST API server. + """ + + _max_retries: int = 2 + _timeout: int = 10 + _base_url: str = None + _token: str = None + _username: str = None + + def __init__(self, base_url: str, token: str = None, username: str = None): + """ + Instantiates a CleverSwarmClient instance. + :param base_url: the base_url of the REST API server + :param token: a token if already available, possibly avoiding a login step if the token is still valid + :param username: the username for which the login should be performed, if not specified, it will be asked + """ + self._base_url = base_url + self._token = token + self._username = username + self._triplets_template = "{}_output.jsonl" + + def _generic_http_request_executor( + self, blogic_method_wrapper, req_name: str = "request", is_login: bool = False + ): + """ + Helper method to execute + :param blogic_method_wrapper: the wrapper function to the actual business logic + :param req_name: a text that summarizes the request to be invoked + :param is_login: True, indicates it is a login request + :return: the blogic_method_wrapper return value + """ + retries = 0 + relogin = False + while retries <= self._max_retries: + try: + if not is_login and relogin: + self.update_token() + relogin = False + + return blogic_method_wrapper() + except Exception as error: + if isinstance(error, ClientException): + if retries < self._max_retries and isinstance( + error, UnauthorizedException + ): + if ( + str(error).lower().strip() + == "user provided credentials are not correct." + ): + relogin = True + elif retries < self._max_retries and isinstance( + error, InvalidUsernameOrPasswordException + ): + relogin = True + else: + raise error + elif isinstance(error, requests.exceptions.HTTPError): + logger.error( + f"Failed to execute {req_name} with cause {error}", + exc_info=True, + ) + raise RequestErrorException( + "Failed process request, error from server." + ) + elif isinstance(error, requests.exceptions.ConnectionError): + logger.error( + f"Failed to execute {req_name} due connection error {error}", + exc_info=True, + ) + raise ConnectionErrorException("Failed to connect to server.") + elif isinstance(error, requests.exceptions.ConnectTimeout): + logger.error( + f"Failed to execute {req_name} due connection timeout {error}", + exc_info=True, + ) + raise ConnectionTimeoutException( + "Communication with server timed out." + ) + elif isinstance(error, requests.exceptions.ReadTimeout): + logger.error( + f"Failed to execute {req_name} due read timeout {error}", + exc_info=True, + ) + raise ConnectionTimeoutException( + "Communication timed out while receiving data from the server." + ) + else: + logger.error( + f"Failed to execute {req_name} due to unexpected error {error}", + exc_info=True, + ) + raise UnexpectedConditionException( + "Client error while processing authorization" + ) + retries += 1 + + def update_token(self, query_username: bool = False) -> Self: + """ + Updates the session authorization token. + :return: the BenchmarkClient instance + """ + if self._base_url is None or self._base_url.strip() == "": + raise BaseUrlMissingException( + "Cannot update token, because no URL base was provided." + ) + + def func_get_updated_token(): + if self._username is None or query_username: + self._username = input("Please enter your username: ") + password = getpass("Please enter your password: ") + # TODO Hash password + data = {"username": self._username, "password": password} + + r = requests.post( + self._base_url + "login", data=data, timeout=self._timeout + ) + if r.status_code == 400: + raise InvalidUsernameOrPasswordException( + "Invalid username or password entered." + ) + else: + r.raise_for_status() + + token_data = r.json() + if token_data["token_type"].lower() != "bearer": + logger.error( + "Unexpected token_type: {}".format(token_data["token_type"]) + ) + raise UnexpectedConditionException( + "Unexpected token type on authorization" + ) + + self._token = token_data["access_token"] + return None + + self._generic_http_request_executor( + func_get_updated_token, "authorization request", True + ) + + return self + + def get_jobs_list(self, is_benchmark: bool = True) -> List: + """ + Obtains the list of jobs of the specified type in the server along with their statuses. + Login is requested if required by the server. + :param is_benchmark: True, if only benchmark jobs are to be returned. + False, if only regular text to KG extraction jobs are to be returned. + :return: the list of benchmark jobs + """ + if self._token is None: + self.update_token() + + def func_get_jobs_list(): + headers = {"Authorization": "Bearer " + self._token} + response = requests.get( + self._base_url + "jobs", headers=headers, timeout=self._timeout + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + + test = None + if is_benchmark: + + def test_is_bench(x: JobType) -> bool: + return x == JobType.Benchmark + + test = test_is_bench + else: + + def test_is_not_bench(x: JobType) -> bool: + return x != JobType.Benchmark + + test = test_is_not_bench + + # Filter response to only return Benchmark jobs + results = [] + response_json = response.json() + for result in response_json: + if test(JobType(result["type"])): + results.append(result) + return results + + return self._generic_http_request_executor( + func_get_jobs_list, "get jobs list request" + ) + + def get_job_status(self, job_id: str) -> JobStatus: + """ + Obtains the job status. + Login is requested if required by the server. + :param job_id: the job id + :return: the current job status in the server + """ + if self._token is None: + self.update_token() + + def func_get_job_status(): + headers = {"Authorization": "Bearer " + self._token} + response = requests.get( + self._base_url + "jobs/" + job_id, + headers=headers, + timeout=self._timeout, + ) + + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif e.response.status_code == 404: + raise JobNotFoundException( + "Job with id: {} not found.".format(job_id) + ) + else: + logger.error(f"Error retrieving job details: {e}") + raise RequestErrorException(f"Failed to retrieve job details: {e}") + + logger.debug(f"Server responded: {response.json()}") + return response.json() + + status_str: str = self._generic_http_request_executor( + func_get_job_status, "get job status request" + ) + return JobStatus(status_str) + + def get_job_details(self, job_id: str) -> Dict[str, Any]: + """ + Obtains the job details including input files and status. + Login is requested if required by the server. + :param job_id: the job id + :return: the job details dictionary + """ + if self._token is None: + self.update_token() + + def func_get_job_details(): + headers = {"Authorization": "Bearer " + self._token} + response = requests.get( + self._base_url + "jobs/details/" + job_id, + headers=headers, + timeout=self._timeout, + ) + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif e.response.status_code == 404: + raise JobNotFoundException( + "Job with id: {} not found.".format(job_id) + ) + else: + logger.error(f"Error retrieving job details: {e}") + raise RequestErrorException(f"Failed to retrieve job details: {e}") + + logger.debug(f"Server responded: {response.json()}") + return response.json() + + details_str = self._generic_http_request_executor( + func_get_job_details, "get job details request" + ) + return details_str + + def delete_job(self, job_id: str): + """ + Deletes a given job by its job_id. + Login is requested if required by the server. + :param job_id: the job id of the job to be deleted + :return: True, if the job was successfully deleted, False, otherwise. + """ + if self._token is None: + self.update_token() + + def func_delete_job(): + headers = {"Authorization": "Bearer " + self._token} + response = requests.delete( + self._base_url + "jobs/" + job_id, + headers=headers, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException(response.json()["detail"]) + elif response.status_code == 404: + raise JobNotFoundException(f"Job with ID: {job_id} was not found.") + elif response.status_code == 412: + return False + + return True + + return self._generic_http_request_executor( + func_delete_job, "delete job request" + ) + + def retry_job(self, job_id: str) -> JobStatus: + """ + Retry a given job specified by its job_id. Only jobs in Failed state can be retried. + :param job_id: the job id of the job to be retried + :return: the new job status and configuration + """ + if self._token is None: + self.update_token() + + def func_retry_job(): + headers = {"Authorization": "Bearer " + self._token} + response = requests.put( + self._base_url + "jobs/" + job_id, + headers=headers, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif response.status_code == 404: + raise JobNotFoundException(f"Job with ID: {job_id} was not found.") + elif response.status_code == 412: + raise ClientException( + "Cannot retry job because it is not in Failed state." + ) + + logger.debug(f"Server responded: {response.json()}") + return response.json() + + return self._generic_http_request_executor(func_retry_job, "retry job request") + + def poll_job_ready_or_failed(self, job_id: str) -> JobStatus: + """ + Polls a given server job and keeps polling until the job either completes, or fails. + Login is requested if required by the server. + :param job_id: the job id of the server job to be polled + :return: the final status of the job at the end of the poll + """ + status: JobStatus = self.get_job_status(job_id) + while status != JobStatus.Failed and status != JobStatus.Completed: + logger.info(f"Polling - Current status is: {status}") + time.sleep(20) + status = self.get_job_status(job_id) + logger.info(f"Polling - Current status is: {status}") + return status + + def create_benchmark_job( + self, + test_file: Path, + ontology_file: Path, + ground_truth_file: Path, + last_fileset=True, + ): + """ + Creates a benchmark job in the server and uploads the files required. + Login is requested if required by the server. + If the upload is successful, it returns a job ID. + :param test_file: a valid file path containing test cases for benchmark + :param ontology_file: a valid file containing an ontology in JSON format. + :param ground_truth_file: a valid file path containing ground truth data. + :param last_fileset: True, indicates this is the last file set, so the job can be placed in the ready queue. + :return: the job ID where the files were placed. + """ + if self._token is None: + self.update_token() + + if not test_file.exists() or not test_file.is_file(): + raise InvalidFileException( + f"Input test file does not exist or is not a file: {str(test_file)}" + ) + + if not ontology_file.exists() or not ontology_file.is_file(): + raise InvalidFileException( + f"Ontology file does not exist or is not a file: {str(ontology_file)}" + ) + + if not ground_truth_file.exists() or not ground_truth_file.is_file(): + raise InvalidFileException( + f"Ground truth file does not exist or is not a file: {str(ground_truth_file)}" + ) + + def func_create_benchmark_job(): + headers = {"Authorization": "Bearer " + self._token} + params = {"ready_for_processing": last_fileset} + files = [ + ("unstructured", open(test_file, "rb")), + ("ontology", open(ontology_file, "rb")), + ("ground_truth", open(ground_truth_file, "rb")), + ] + response = requests.post( + self._base_url + "benchmark", + headers=headers, + params=params, + files=files, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif response.status_code == 400: + raise InvalidFileException( + response.json().get("detail", "Invalid File") + ) + logger.debug(f"Server responded: {response.json()}") + return response.json()["job_id"] + + return self._generic_http_request_executor( + func_create_benchmark_job, "create benchmark request" + ) + + def append_to_benchmark_job( + self, + job_id: str, + test_file: Path, + ontology_file: Path, + ground_truth_file: Path, + last_fileset=True, + ): + """ + Appends a new fileset to an already existing benchmark job. + Login is requested if required by the server. + :param job_id: an existing job id that accepts this file set + :param test_file: the test file path + :param ontology_file: the ontology file path + :param ground_truth_file: the ground truth file + :param last_fileset: True, indicates this is the last file set, so the job can be placed in the ready queue. + :return: the job id of the job + """ + if self._token is None: + self.update_token() + + if not test_file.exists() or not test_file.is_file(): + raise InvalidFileException( + f"Input test file does not exist or is not a file: {str(test_file)}" + ) + + if not ontology_file.exists() or not ontology_file.is_file(): + raise InvalidFileException( + f"Ontology file does not exist or is not a file: {str(ontology_file)}" + ) + + if not ground_truth_file.exists() or not ground_truth_file.is_file(): + raise InvalidFileException( + f"Ground truth file does not exist or is not a file: {str(ground_truth_file)}" + ) + + def func_append_to_benchmark_job(): + headers = {"Authorization": "Bearer " + self._token} + params = {"ready_for_processing": last_fileset} + files = [ + ("unstructured", open(test_file, "rb")), + ("ontology", open(ontology_file, "rb")), + ("ground_truth", open(ground_truth_file, "rb")), + ] + response = requests.put( + self._base_url + f"benchmark/{job_id}", + headers=headers, + params=params, + files=files, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif response.status_code == 400: + raise InvalidFileException( + response.json().get("detail", "Invalid File") + ) + elif response.status_code == 404: + raise JobNotFoundException(f"Job ID: {job_id} does not exist.") + + logger.debug(f"Server responded: {response.json()}") + return response.json() + + return self._generic_http_request_executor( + func_append_to_benchmark_job, "append to benchmark request" + ) + + def retrieve_benchmark_files( + self, job_id: str, output_filepaths: List[Path] + ) -> List[Path]: + """ + Retrieves the files containing the benchmark results from the server. + Login is requested if required by the server. + :param job_id: the benchmark job id + :param output_filepaths: the output filepaths where the benchmark files will be stored. + :return: a list of file paths where the benchmark files were stored. The order of the files is the same as + they were uploaded. + """ + if self._token is None: + self.update_token() + + output_filenames: List[Path] = [] + for idx in range(len(output_filepaths)): + try: + os.makedirs(output_filepaths[idx].parent.as_posix(), exist_ok=True) + except OSError as error: + error_path = str(output_filepaths[idx].parent) + logger.log( + logging.CRITICAL, + f"Failed to create folder: {error_path} with cause {error}", + exc_info=True, + ) + raise UnexpectedConditionException( + f"Could not create output folder: {error_path}" + ) + + def func_retrieve_benchmark_file(): + headers = {"Authorization": "Bearer " + self._token} + params = {"index": idx} + response = requests.get( + self._base_url + "benchmark/" + job_id, + headers=headers, + params=params, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + if response.status_code == 204: + raise ClientException( + f"Benchmark job {job_id} not ready at index {idx}" + ) + if response.status_code == 404: + raise JobNotFoundException( + f"Benchmark output not found for job {job_id}, index {idx}" + ) + + with open(output_filepaths[idx], "wb") as fd: + for chunk in response.iter_content( + chunk_size=512 * 1024 + ): # 512 kBytes chunks + fd.write(chunk) + + return output_filepaths[idx] + + filepath: Path = self._generic_http_request_executor( + func_retrieve_benchmark_file, "retrieve benchmark file request" + ) + output_filenames.append(filepath) + + return output_filenames + + def create_unstructured_to_kg_job( + self, + unstructured_text_file: Path, + ontology_file: Path, + ontology_spec_file: Path, + force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect, + ): + """ + Creates a text to KG conversion job in the server and uploads the required files. + Login is requested if required by the server. + If the upload is successful, it returns a job ID. + :param unstructured_text_file: a valid file path containing unstructured text for conversion + :param ontology_file: a valid file containing an ontology in JSON format. + :param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format. + :param force_filetype: indicate the file type to force for the input file + :return: the job ID where the files were placed. + """ + if self._token is None: + self.update_token() + + if not unstructured_text_file.exists() or not unstructured_text_file.is_file(): + raise InvalidFileException( + f"Unstructured text file does not exist or is not a file: " + f"{str(unstructured_text_file)}" + ) + + if not ontology_file.exists() or not ontology_file.is_file(): + raise InvalidFileException( + f"Ontology files does not exist or is not a file: {str(ontology_file)}" + ) + + if not ontology_spec_file.exists() or not ontology_spec_file.is_file(): + raise InvalidFileException( + f"OWL/XML ontology file does not exist or is not a file: " + f"{str(ontology_spec_file)}" + ) + + def func_create_unstructured_to_kg_job(): + headers = {"Authorization": "Bearer " + self._token} + params = {"file_type": force_filetype} + files = [ + ("unstructured", open(unstructured_text_file, "rb")), + ("ontology", open(ontology_file, "rb")), + ("ontology_spec", open(ontology_spec_file, "rb")), + ] + response = requests.post( + self._base_url + "unstructured/with_ontology", + headers=headers, + params=params, + files=files, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unknown reason") + ) + elif response.status_code == 400: + raise InvalidFileException( + response.json().get("detail", "Unknown reason") + ) + + if response.status_code == 201: + return response.json()["job_id"] + else: + response.raise_for_status() + response_data = response.json() + logger.debug(f"Server response: {response_data}") + return response.json()["job_id"] + + return self._generic_http_request_executor( + func_create_unstructured_to_kg_job, + "create unstructured to KG extraction job request", + ) + + def create_unstructured_to_kg_wildcards_job( + self, + unstructured_text_file: Path, + ontology_file: Path, + ontology_spec_file: Path, + wildcards_query_file: Path, + force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect, + ): + """ + Creates a text to KG conversion job (with wildcards filtering) in the server and uploads the required files. + Login is requested if required by the server. + If the upload is successful, it returns a job ID. + :param unstructured_text_file: a valid file path containing unstructured text for conversion + :param ontology_file: a valid file containing an ontology in JSON format. + :param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format. + :param wildcards_query_file: a valid file path containing wildcard queries in JSON. + :param force_filetype: indicate the file type to force for the input file + :return: the job ID where the files were placed. + """ + if self._token is None: + self.update_token() + + if not unstructured_text_file.exists() or not unstructured_text_file.is_file(): + raise InvalidFileException( + f"Unstructured text file does not exist or is not a file: " + f"{str(unstructured_text_file)}" + ) + + if not ontology_file.exists() or not ontology_file.is_file(): + raise InvalidFileException( + f"Ontology files does not exist or is not a file: {str(ontology_file)}" + ) + + if not ontology_spec_file.exists() or not ontology_spec_file.is_file(): + raise InvalidFileException( + f"OWL/XML ontology file does not exist or is not a file: " + f"{str(ontology_spec_file)}" + ) + + if not wildcards_query_file.exists() or not wildcards_query_file.is_file(): + raise InvalidFileException( + f"Wildcards query file does not exist or is not a file: " + f"{str(wildcards_query_file)}" + ) + + def func_create_unstructured_to_kg_wildcards_job(): + headers = {"Authorization": "Bearer " + self._token} + params = {"file_type": force_filetype} + files = [ + ("unstructured", open(unstructured_text_file, "rb")), + ("ontology", open(ontology_file, "rb")), + ("ontology_spec", open(ontology_spec_file, "rb")), + ("wildcards", open(wildcards_query_file, "rb")), + ] + response = requests.post( + self._base_url + "unstructured/with_wildcards", + headers=headers, + params=params, + files=files, + timeout=self._timeout, + ) + if response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unknown reason") + ) + elif response.status_code == 400: + raise InvalidFileException( + response.json().get("detail", "Unknown reason") + ) + + if response.status_code == 201: + return response.json()["job_id"] + else: + response.raise_for_status() + response_data = response.json() + logger.debug(f"Server response: {response_data}") + return response.json()["job_id"] + + return self._generic_http_request_executor( + func_create_unstructured_to_kg_wildcards_job, + "create unstructured to KG extraction with wildcards job request", + ) + + def retrieve_unstructured_to_kg_files( + self, job_id: str, output_filepath: Path + ) -> Path: + """ + Retrieves the files containing the unstructured to KG results from the server. + Login is requested if required by the server. + :param job_id: the unstructured to kg job id + :param output_filepath: the output filepath where the KG triplet file will be stored. + :return: a file path where the KG file was stored. + """ + if self._token is None: + self.update_token() + + # Ensure output directory exists, but only if parent directory already exists, otherwise it can indicate a + # configuration issue. + try: + os.makedirs(output_filepath.parent.as_posix(), exist_ok=True) + except OSError as error: + error_path = str(output_filepath.parent) + logger.log( + logging.CRITICAL, + f"Failed to create folder: {error_path} with cause {error}", + exc_info=True, + ) + raise UnexpectedConditionException( + f"Could not create output folder: {error_path}" + ) + + def func_retrieve_unstructured_to_kg_file(): + headers = {"Authorization": "Bearer " + self._token} + params = {"response_type": ResponseTypeAPI.JsonFile} + response = requests.get( + self._base_url + "unstructured/" + job_id, + headers=headers, + params=params, + timeout=self._timeout, + ) + # Check response status + if response.status_code == 204: + raise ClientException(f"Job results not ready yet for job ID: {job_id}") + elif response.status_code == 401: + raise UnauthorizedException( + response.json().get("detail", "Unauthorized") + ) + elif response.status_code == 404: + raise JobNotFoundException(f"Job results for ID: {job_id} not found") + + # Raise exception for any other non-success status code + response.raise_for_status() + + # Save the content to the output file + with open(output_filepath, "wb") as fd: + for chunk in response.iter_content( + chunk_size=512 * 1024 + ): # 512 kBytes chunks + fd.write(chunk) + + logger.info(f"Successfully wrote results to {output_filepath}") + return output_filepath + + filepath: Path = self._generic_http_request_executor( + func_retrieve_unstructured_to_kg_file, + "retrieve unstructured to KG file request", + ) + return filepath diff --git a/src/cleverswarm_python_client/libs/evaluation/__init__.py b/src/cleverswarm_python_client/libs/evaluation/__init__.py new file mode 100644 index 0000000..e22d02e --- /dev/null +++ b/src/cleverswarm_python_client/libs/evaluation/__init__.py @@ -0,0 +1 @@ +"""Evaluation utilities for text2kg package.""" diff --git a/src/cleverswarm_python_client/libs/evaluation/detailed_eval.py b/src/cleverswarm_python_client/libs/evaluation/detailed_eval.py new file mode 100644 index 0000000..fb73cbf --- /dev/null +++ b/src/cleverswarm_python_client/libs/evaluation/detailed_eval.py @@ -0,0 +1,399 @@ +""" +This module provides functionality for detailed evaluation of entity and relation extraction. +""" + +import json +import os +import re +from collections import defaultdict +from typing import Dict, List, Set, Tuple + + +class CustomEncoder(json.JSONEncoder): + """Custom JSON encoder to handle sets.""" + + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return super().default(obj) + + +def read_jsonl(file_path: str) -> List[Dict]: + """Read a JSONL file and return its contents as a list of dictionaries.""" + with open(file_path, "r", encoding="utf-8") as f: + return [json.loads(line) for line in f] + + +def write_jsonl(data: List[Dict], file_path: str): + """Write a list of dictionaries to a JSONL file.""" + with open(file_path, "w", encoding="utf-8") as f: + for item in data: + json.dump(item, f, cls=CustomEncoder) + f.write("\n") + + +def load_ontology(ontology_path: str) -> Dict: + """Load an ontology from a JSON file.""" + with open(ontology_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def calculate_metrics(tp: int, fp: int, fn: int) -> Tuple[float, float, float]: + """Calculate precision, recall, and F1 score.""" + precision = tp / (tp + fp) if tp + fp > 0 else 0 + recall = tp / (tp + fn) if tp + fn > 0 else 0 + f1 = ( + 2 * (precision * recall) / (precision + recall) if precision + recall > 0 else 0 + ) + return round(precision, 2), round(recall, 2), round(f1, 2) + + +def normalize_string(label: str) -> str: + """Normalize a string by removing special characters and converting to lowercase.""" + # Remove everything in parentheses including parentheses + label = re.sub(r"\(.*?\)", "", label) + # Remove everything after a comma including the comma + label = re.sub(r",.*", "", label) + # Removes occurences of 'the' and subsequent white-spaces + label = re.sub(r"^(The|the)\s+", "", label) + # Remove every non-alphanumeric character and spaces + label = re.sub(r"[^a-zA-Z0-9]", "", label) + # Remove alphabets that appear after numbers + label = re.sub(r"(\d+)[a-zA-Z]+", r"\1", label) + # Remove trailing 's' + label = re.sub(r"s$", "", label) + # Convert to lowercase + return label.lower() + + +def normalize_triple(sub_label: str, rel_label: str, obj_label: str) -> str: + """ + Normalize triples for comparison in precision, recall calculations + :param sub_label: subject string + :param rel_label: relation string + :param obj_label: object string + :return: a normalized triple as a single concatenated string + """ + # remove spaces and underscores and make lower case + if not isinstance(obj_label, str): + obj_label = str(obj_label) + if obj_label is None: + obj_label = "" + sub_label = normalize_string(sub_label) + rel_label = normalize_string(rel_label) + obj_label = normalize_string(obj_label) + # concatenate them to a single string + tr_key = f"{sub_label}{rel_label}{obj_label}" + return tr_key + + +def evaluate_entity_presence(ground_truth: Dict, response: Dict) -> Dict: + """Evaluate the presence of entities in the response compared to ground truth.""" + gt_entities = set() + for triple in ground_truth["triples"]: + gt_entities.add(normalize_string(triple["sub"])) + gt_entities.add(normalize_string(triple["obj"])) + + response_entities = set(normalize_string(ent["text"]) for ent in response["ents"]) + + tp = len(gt_entities.intersection(response_entities)) + fp = len(response_entities - gt_entities) + fn = len(gt_entities - response_entities) + + precision, recall, f1 = calculate_metrics(tp, fp, fn) + + return { + "precision": precision, + "recall": recall, + "f1": f1, + "has_entities": len(response_entities) > 0, + } + + +def evaluate_entity_linking(ground_truth: Dict, response: Dict, ontology: Dict) -> Dict: + """Evaluate the accuracy of entity linking.""" + class_mapping = {} + for relation, class_pairs in ontology["Relations"].items(): + for domain, range_class in class_pairs: + for triple in ground_truth["triples"]: + if triple["rel"] == relation: + class_mapping[normalize_string(triple["sub"])] = domain + class_mapping[normalize_string(triple["obj"])] = range_class + + correct_links = 0 + total_links = 0 + gt_class_instances = [] + correctly_linked_entities = set() + + for ent in response["ents"]: + normalized_text = normalize_string(ent["text"]) + if normalized_text in class_mapping: + total_links += 1 + gt_class = class_mapping[normalized_text] + gt_class_instances.append( + { + "text": ent["text"], + "gt_class": gt_class, + "response_class": ent["class"], + } + ) + if ent["class"] == gt_class: + correct_links += 1 + correctly_linked_entities.add(normalized_text) + + accuracy = correct_links / total_links if total_links > 0 else None + + return { + "accuracy": round(accuracy, 2) if accuracy is not None else None, + "correct_links": correct_links, + "total_links": total_links, + "gt_class_instances": gt_class_instances, + "correctly_linked_entities": list(correctly_linked_entities), + "has_links": total_links > 0, + } + + +def evaluate_relation_linking( + ground_truth: Dict, response: Dict, correctly_linked_entities: List[str] +) -> Dict: + """Evaluate the accuracy of relation linking.""" + gt_triples = ground_truth["triples"] + response_triples = response["triples"] + + filtered_gt_triples = [ + triple + for triple in gt_triples + if normalize_string(triple["sub"]) in correctly_linked_entities + and normalize_string(triple["obj"]) in correctly_linked_entities + ] + + gt_dict = { + (normalize_string(triple["sub"]), normalize_string(triple["obj"])): triple[ + "rel" + ] + for triple in filtered_gt_triples + } + response_dict = { + (normalize_string(triple[0]), normalize_string(triple[2])): triple[1] + for triple in response_triples + } + + correct_triples = [] + missing_triples = [] + wrong_triples = [] + + for (sub, obj), rel in gt_dict.items(): + if (sub, obj) in response_dict: + if response_dict[(sub, obj)] == rel: + correct_triples.append((sub, rel, obj)) + else: + wrong_triples.append((sub, response_dict[(sub, obj)], obj)) + else: + missing_triples.append((sub, rel, obj)) + + total_triples = len(filtered_gt_triples) + + return { + "correct": ( + round(len(correct_triples) / total_triples, 2) + if total_triples > 0 + else None + ), + "missing": ( + round(len(missing_triples) / total_triples, 2) + if total_triples > 0 + else None + ), + "wrong": ( + round(len(wrong_triples) / total_triples, 2) if total_triples > 0 else None + ), + "correct_triples": correct_triples, + "missing_triples": missing_triples, + "wrong_triples": wrong_triples, + "total_triples": total_triples, + "has_triples": total_triples > 0, + } + + +def evaluate_triples(ground_truth: Dict, response: Dict) -> Dict: + """Evaluate the accuracy of relation linking.""" + gt_triples = ground_truth["triples"] + response_triples = response["triples"] + + # collect the set of relations in ground truth triples, spaces are converted to "_" to make them + # comparable with system triples + gt_relations = {tr["rel"].replace(" ", "_") for tr in gt_triples} + + # filter out any triples in system output that does not match with ground truth relations + filtered_system_triples = [tr for tr in response_triples if tr[1] in gt_relations] + + # create a normalized string from subject, relation, object of each triple for comparison + normalized_system_triples = { + normalize_triple(tr[0], tr[1], tr[2]) for tr in filtered_system_triples + } + normalized_gt_triples = { + normalize_triple(tr["sub"], tr["rel"], tr["obj"]) for tr in gt_triples + } + + # compare the system output triples with ground truth triples and calculate precision, recall, f1 + precision, recall, f1 = calculate_precision_recall_f1( + normalized_gt_triples, normalized_system_triples + ) + + return {"triples_precision": precision, "triples_recall": recall, "triples_f1": f1} + + +def evaluate_ontology(ontology_id: str, config: Dict) -> Dict: + """Evaluate the performance for a specific ontology.""" + output_path = config["path_patterns"]["sys"].replace("$$onto$$", ontology_id) + ground_truth_path = config["path_patterns"]["gt"].replace("$$onto$$", ontology_id) + ontology_path = config["path_patterns"]["onto"].replace("$$onto$$", ontology_id) + + outputs = read_jsonl(output_path) + ground_truths = read_jsonl(ground_truth_path) + ontology = load_ontology(ontology_path) + + results = [] + + for gt, resp in zip(ground_truths, outputs): + triples_evaluation = evaluate_triples(gt, resp) + entity_presence = evaluate_entity_presence(gt, resp) + entity_linking = evaluate_entity_linking(gt, resp, ontology) + relation_linking = evaluate_relation_linking( + gt, resp, entity_linking["correctly_linked_entities"] + ) + + results.append( + { + "id": gt["id"], + "triples_evaluation": triples_evaluation, + "entity_presence": entity_presence, + "entity_linking": entity_linking, + "relation_linking": relation_linking, + "response_entities": resp["ents"], + "response_triples": resp["triples"], + "ground_truth_triples": gt["triples"], + "sent": gt["sent"], + } + ) + + return results + + +def calculate_average_metrics(results: List[Dict], is_detailed_metrics: bool) -> Dict: + """Calculate average metrics across all results.""" + avg_metrics = defaultdict(lambda: {"sum": 0, "count": 0}) + for result in results: + teval = result["triples_evaluation"] + ep = result["entity_presence"] + if is_detailed_metrics: + el = result["entity_linking"] + rl = result["relation_linking"] + + avg_metrics["triples_evaluation_precision"]["sum"] += teval["triples_precision"] + avg_metrics["triples_evaluation_recall"]["sum"] += teval["triples_recall"] + avg_metrics["triples_evaluation_f1"]["sum"] += teval["triples_f1"] + avg_metrics["triples_evaluation_precision"]["count"] += 1 + avg_metrics["triples_evaluation_recall"]["count"] += 1 + avg_metrics["triples_evaluation_f1"]["count"] += 1 + + avg_metrics["entity_presence_precision"]["sum"] += ep["precision"] + avg_metrics["entity_presence_recall"]["sum"] += ep["recall"] + avg_metrics["entity_presence_f1"]["sum"] += ep["f1"] + avg_metrics["entity_presence_precision"]["count"] += 1 + avg_metrics["entity_presence_recall"]["count"] += 1 + avg_metrics["entity_presence_f1"]["count"] += 1 + + if is_detailed_metrics: + if el["has_links"]: + avg_metrics["entity_linking_accuracy"]["sum"] += el["accuracy"] + avg_metrics["entity_linking_accuracy"]["count"] += 1 + + if rl["has_triples"]: + avg_metrics["relation_linking_correct"]["sum"] += rl["correct"] + avg_metrics["relation_linking_missing"]["sum"] += rl["missing"] + avg_metrics["relation_linking_wrong"]["sum"] += rl["wrong"] + avg_metrics["relation_linking_correct"]["count"] += 1 + avg_metrics["relation_linking_missing"]["count"] += 1 + avg_metrics["relation_linking_wrong"]["count"] += 1 + + return { + key: round(value["sum"] / value["count"], 2) if value["count"] > 0 else None + for key, value in avg_metrics.items() + } + + +def calculate_precision_recall_f1(gold: Set, pred: Set) -> tuple[float, float, float]: + """ + Method to calculate precision, recall and f1: + Precision is calculated as correct_triples/predicted_triples and + Recall as correct_triples/gold_triples + F1 as the harmonic mean of precision and recall. + :param gold: items in the gold standard + :param pred: items in the system prediction + :return: + p: float - precision + r: float - recall + f1: float - F1 + """ + if len(pred) == 0: + return 0, 0, 0 + p = len(gold.intersection(pred)) / len(pred) + r = len(gold.intersection(pred)) / len(gold) + if p + r > 0: + f1 = 2 * ((p * r) / (p + r)) + else: + f1 = 0 + return p, r, f1 + + +def EvaluateResults(config: Dict, is_detailed_metrics: bool = True): + ontology_ids = config["onto_list"] + overall_avg_metrics = defaultdict(list) + + for ontology_id in ontology_ids: + results = evaluate_ontology(ontology_id, config) + new_results = results + if not is_detailed_metrics: + new_results = [] + for result in results: + new_results.append( + { + "id": result["id"], + "entity_presence": result["entity_presence"], + "response_entities": result["response_entities"], + "sent": result["sent"], + } + ) + # all_ontology_results[ontology_id] = new_results + + # Write detailed results for each ontology + output_path = config["path_patterns"]["output"].replace("$$onto$$", ontology_id) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + write_jsonl(new_results, output_path) + + # Calculate average metrics for each ontology + avg_metrics = calculate_average_metrics(results, is_detailed_metrics) + for key, value in avg_metrics.items(): + if value is not None: + overall_avg_metrics[key].append((ontology_id, value)) + + # Calculate overall average metrics and write all results to a single file + overall_avg_output_path = config["overall_output"] + os.makedirs(os.path.dirname(overall_avg_output_path), exist_ok=True) + with open(overall_avg_output_path, "w", encoding="utf-8") as f: + json.dump( + { + "per_ontology_metrics": { + key: {ont_id: value for ont_id, value in values} + for key, values in overall_avg_metrics.items() + }, + "global_avg_metrics": { + key: round(sum(v for _, v in values) / len(values), 2) + for key, values in overall_avg_metrics.items() + }, + }, + f, + indent=2, + cls=CustomEncoder, + ) diff --git a/src/cleverswarm_python_client/libs/exceptions.py b/src/cleverswarm_python_client/libs/exceptions.py new file mode 100644 index 0000000..53778c2 --- /dev/null +++ b/src/cleverswarm_python_client/libs/exceptions.py @@ -0,0 +1,98 @@ +""" +Module that contains Exceptions that maybe thrown by the Benchmark client. + + Copyright (c) 2016 - present Syncleus, Inc. + Copyright (c) 2016 - present CleverThis, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + + +class ClientException(Exception): + """ + Client exception occurred. + """ + + pass + + +class BaseUrlMissingException(ClientException): + """ + Base URL is missing. No REST operations are possible. + """ + + pass + + +class InvalidUsernameOrPasswordException(ClientException): + """ + Invalid username or password. + """ + + pass + + +class UnexpectedConditionException(ClientException): + """ + Something unexpected happened during processing. Some unhandled case that should not occur. + """ + + pass + + +class RequestErrorException(ClientException): + """ + Request is badly formatted or server error occurred. + """ + + pass + + +class ConnectionErrorException(ClientException): + """ + Failed to connect to server. + """ + + pass + + +class ConnectionTimeoutException(ClientException): + """ + Connection to server timed out. + """ + + pass + + +class JobNotFoundException(ClientException): + """ + Job not found. + """ + + pass + + +class UnauthorizedException(ClientException): + """ + Unauthorized exception. Either user is not logged in or is not authorized to do the operation. + """ + + pass + + +class InvalidFileException(ClientException): + """ + File is not valid. + """ + + pass diff --git a/src/cleverswarm_python_client/libs/file_type_enum.py b/src/cleverswarm_python_client/libs/file_type_enum.py new file mode 100644 index 0000000..e5492ca --- /dev/null +++ b/src/cleverswarm_python_client/libs/file_type_enum.py @@ -0,0 +1,14 @@ +from enum import StrEnum + + +class FileTypeAPI(StrEnum): + AutoDetect = "AutoDetect" + Markdown = "Markdown" + PlainText = "PlainText" + + +class FileType(StrEnum): + JSONL = "JSONL" + Markdown = "Markdown" + PlainText = "PlainText" + Unknown = "Unknown" diff --git a/src/cleverswarm_python_client/libs/job_enums.py b/src/cleverswarm_python_client/libs/job_enums.py new file mode 100644 index 0000000..d183ec2 --- /dev/null +++ b/src/cleverswarm_python_client/libs/job_enums.py @@ -0,0 +1,34 @@ +""" +Module that contains the enumerations found when interfacing with Benchmark client and server. + + Copyright (c) 2016 - present Syncleus, Inc. + Copyright (c) 2016 - present CleverThis, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +from enum import StrEnum + + +class JobType(StrEnum): + UnstructuredWithOntology = "UnstructuredWithOntology" + UnstructuredWithWildcards = "UnstructuredWithWildcards" + Benchmark = "Benchmark" + + +class JobStatus(StrEnum): + Created = "Created" + ReadyForProcessing = "ReadyForProcessing" + Processing = "Processing" + Completed = "Completed" + Failed = "Failed" diff --git a/src/cleverswarm_python_client/libs/response_type_enum.py b/src/cleverswarm_python_client/libs/response_type_enum.py new file mode 100644 index 0000000..fa52d94 --- /dev/null +++ b/src/cleverswarm_python_client/libs/response_type_enum.py @@ -0,0 +1,6 @@ +from enum import StrEnum + + +class ResponseTypeAPI(StrEnum): + JsonText = "JsonText" + JsonFile = "JsonFile" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2450865 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,144 @@ +""" +Pytest configuration and shared fixtures for CleverSwarm Python client tests. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +import pytest + + +@pytest.fixture +def temp_directory(): + """Create a temporary directory for testing.""" + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + + +@pytest.fixture +def sample_text_file(temp_directory): + """Create a sample text file for testing.""" + text_file = temp_directory / "sample.txt" + text_file.write_text("This is a sample text file for testing purposes.") + return text_file + + +@pytest.fixture +def sample_json_file(temp_directory): + """Create a sample JSON file for testing.""" + json_file = temp_directory / "sample.json" + json_file.write_text('{"test": "data", "ontology": "sample"}') + return json_file + + +@pytest.fixture +def sample_owl_file(temp_directory): + """Create a sample OWL file for testing.""" + owl_file = temp_directory / "sample.owl" + owl_file.write_text('') + return owl_file + + +@pytest.fixture +def sample_wildcards_file(temp_directory): + """Create a sample wildcards file for testing.""" + wildcards_file = temp_directory / "sample.wildcards" + wildcards_file.write_text('{"wildcards": "test_data"}') + return wildcards_file + + +@pytest.fixture +def sample_ground_truth_file(temp_directory): + """Create a sample ground truth file for testing.""" + gt_file = temp_directory / "sample_gt.jsonl" + gt_file.write_text('{"id": "test1", "sent": "Test sentence", "triples": [{"sub": "Test", "rel": "is", "obj": "example"}]}') + return gt_file + + +@pytest.fixture +def mock_requests_response(): + """Create a mock requests response for testing.""" + class MockResponse: + def __init__(self, status_code=200, json_data=None, content=None): + self.status_code = status_code + self._json_data = json_data or {} + self._content = content or b'' + + def json(self): + return self._json_data + + @property + def content(self): + return self._content + + return MockResponse + + +@pytest.fixture +def mock_cleverswarm_client(): + """Create a mock CleverSwarmClient for testing.""" + from unittest.mock import Mock + + mock_client = Mock() + mock_client.login.return_value = "test_token" + mock_client.logout.return_value = None + mock_client.is_logged_in.return_value = True + mock_client.create_unstructured_to_kg_job.return_value = "test_job_123" + mock_client.create_unstructured_to_kg_wildcards_job.return_value = "test_job_123" + mock_client.create_benchmark_job.return_value = "test_benchmark_123" + mock_client.append_to_benchmark_job.return_value = True + mock_client.get_job_status.return_value = "Completed" + mock_client.get_job_details.return_value = {"id": "test_job_123", "status": "Completed"} + mock_client.get_jobs_list.return_value = [] + mock_client.delete_job.return_value = True + mock_client.poll_job_ready_or_failed.return_value = "Completed" + mock_client.retrieve_unstructured_to_kg_files.return_value = True + mock_client.retrieve_benchmark_files.return_value = True + + return mock_client + + +@pytest.fixture +def sample_job_data(): + """Create sample job data for testing.""" + return { + "id": "test_job_123", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"] + } + + +@pytest.fixture +def sample_benchmark_job_data(): + """Create sample benchmark job data for testing.""" + return { + "id": "test_benchmark_123", + "created": "2023-01-01T00:00:00Z", + "type": "Benchmark", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["1_university_ontology.json"], + "ground_truth_sources": ["ont_1_university_ground_truth.jsonl"] + } diff --git a/tests/evaluation/test_detailed_eval.py b/tests/evaluation/test_detailed_eval.py new file mode 100644 index 0000000..41b1acc --- /dev/null +++ b/tests/evaluation/test_detailed_eval.py @@ -0,0 +1,333 @@ +import pytest +from cleverswarm_python_client.libs.evaluation.detailed_eval import ( + calculate_metrics, normalize_string, normalize_triple, + evaluate_entity_presence, evaluate_entity_linking, + evaluate_relation_linking, calculate_precision_recall_f1, + evaluate_triples, read_jsonl, write_jsonl, load_ontology, + calculate_average_metrics, evaluate_ontology, CustomEncoder, + EvaluateResults +) +import json +import tempfile +import os + +@pytest.fixture +def sample_ontology(): + return { + "Relations": { + "works_at": [["Person", "Organization"]], + "part_of": [["Organization", "Organization"]], + "located_in": [["Organization", "Location"]] + } + } + +@pytest.fixture +def sample_ground_truth(): + return { + "id": "test1", + "sent": "John works at Apple in California", + "triples": [ + {"sub": "John", "rel": "works_at", "obj": "Apple"}, + {"sub": "Apple", "rel": "located_in", "obj": "California"} + ] + } + +@pytest.fixture +def sample_response(): + return { + "id": "test1", + "ents": [ + {"text": "John", "class": "Person"}, + {"text": "Apple", "class": "Organization"}, + {"text": "California", "class": "Location"} + ], + "triples": [ + ["John", "works_at", "Apple"], + ["Apple", "located_in", "California"] + ] + } + +def test_calculate_metrics(): + # Test normal case + precision, recall, f1 = calculate_metrics(tp=5, fp=2, fn=1) + assert precision == round(5/7, 2) + assert recall == round(5/6, 2) + assert f1 == round(2 * (5/7 * 5/6) / (5/7 + 5/6), 2) + + # Test edge case with zero values + precision, recall, f1 = calculate_metrics(tp=0, fp=0, fn=0) + assert precision == 0 + assert recall == 0 + assert f1 == 0 + +def test_normalize_string(): + test_cases = [ + ("The Test String", "teststring"), + ("Test (with parentheses)", "test"), + ("Test123abc", "test123"), + ("Company, Inc.", "company"), + ("Multiple Spaces", "multiplespace"), + ("Numbers123", "numbers123"), + ("The Company", "company"), + ("Products & Services", "productsservice") + ] + + for input_str, expected in test_cases: + result = normalize_string(input_str) + assert result == expected, f"Failed for input '{input_str}': expected '{expected}' but got '{result}'" + +def test_normalize_triple(): + test_cases = [ + ("Subject", "Relation", "Object", "subjectrelationobject"), + ("The Company", "has_product", "Item", "companyhasproductitem"), + ("Person", "works_at", None, "personworksatnone"), + ("Test", "test", "", "testtest") + ] + + for sub, rel, obj, expected in test_cases: + result = normalize_triple(sub, rel, obj) + assert result == expected, f"Failed for input ({sub}, {rel}, {obj}): expected '{expected}' but got '{result}'" + +def test_normalize_string_edge_cases(): + test_cases = [ + ("", ""), # Empty string + (" ", ""), # Only spaces + ("The", "the"), # Single word with 'The' + ("A B C", "abc"), # Multiple single letters + ("Test123Test", "test123"), # Mixed alphanumeric + ("Test-123_Test", "test123"), # With special characters + ] + + for input_str, expected in test_cases: + result = normalize_string(input_str) + assert result == expected, f"Failed for input '{input_str}': expected '{expected}' but got '{result}'" + +def test_normalize_triple_edge_cases(): + test_cases = [ + ("", "", "", ""), # All empty + ("A", "B", "C", "abc"), # Single letters + ("The A", "The B", "The C", "abc"), # With articles + ("Test(1)", "Test,2", "Test.3", "testtesttest3"), # With special chars and numbers + ] + + for sub, rel, obj, expected in test_cases: + result = normalize_triple(sub, rel, obj) + assert result == expected, f"Failed for input ({sub}, {rel}, {obj}): expected '{expected}' but got '{result}'" + +def test_normalize_triple_none_handling(): + # Test with None as object - this works as the function handles it + result = normalize_triple("Person", "works_at", None) + assert result == "personworksatnone", f"Failed for None object: expected 'personworksatnone' but got '{result}'" + + # Test with empty string instead of None for subject - since function doesn't handle None for subject + result = normalize_triple("", "works_at", "Company") + assert result == "worksatcompany", f"Failed for empty subject: expected 'worksatcompany' but got '{result}'" + +def test_evaluate_entity_presence(sample_ground_truth, sample_response): + result = evaluate_entity_presence(sample_ground_truth, sample_response) + assert isinstance(result, dict) + assert all(key in result for key in ['precision', 'recall', 'f1', 'has_entities']) + assert result['has_entities'] == True + assert result['precision'] > 0 + assert result['recall'] > 0 + assert result['f1'] > 0 + +def test_evaluate_entity_linking(sample_ground_truth, sample_response, sample_ontology): + result = evaluate_entity_linking(sample_ground_truth, sample_response, sample_ontology) + assert isinstance(result, dict) + assert all(key in result for key in ['accuracy', 'correct_links', 'total_links', + 'gt_class_instances', 'correctly_linked_entities', + 'has_links']) + assert result['has_links'] == True + assert result['accuracy'] > 0 + +def test_evaluate_relation_linking(sample_ground_truth, sample_response): + correctly_linked_entities = ["john", "apple", "california"] + result = evaluate_relation_linking(sample_ground_truth, sample_response, correctly_linked_entities) + assert isinstance(result, dict) + assert all(key in result for key in ['correct', 'missing', 'wrong', 'correct_triples', + 'missing_triples', 'wrong_triples', 'total_triples', + 'has_triples']) + assert result['has_triples'] == True + +def test_evaluate_triples(sample_ground_truth, sample_response): + result = evaluate_triples(sample_ground_truth, sample_response) + assert isinstance(result, dict) + assert all(key in result for key in ['triples_precision', 'triples_recall', 'triples_f1']) + assert result['triples_precision'] > 0 + assert result['triples_recall'] > 0 + assert result['triples_f1'] > 0 + +def test_calculate_precision_recall_f1(): + gold = {"triple1", "triple2", "triple3"} + pred = {"triple1", "triple2", "triple4"} + + precision, recall, f1 = calculate_precision_recall_f1(gold, pred) + assert precision == 2/3 + assert recall == 2/3 + assert f1 == 2/3 + + # Test empty prediction + precision, recall, f1 = calculate_precision_recall_f1(gold, set()) + assert precision == 0 + assert recall == 0 + assert f1 == 0 + +def test_jsonl_operations(): + test_data = [{"test": 1}, {"test": 2}] + with tempfile.NamedTemporaryFile(mode='w+', delete=False) as f: + write_jsonl(test_data, f.name) + result = read_jsonl(f.name) + assert result == test_data + os.unlink(f.name) + +def test_calculate_average_metrics(): + results = [ + { + 'triples_evaluation': {'triples_precision': 0.8, 'triples_recall': 0.7, 'triples_f1': 0.75}, + 'entity_presence': {'precision': 0.9, 'recall': 0.85, 'f1': 0.87}, + 'entity_linking': {'accuracy': 0.8, 'has_links': True}, + 'relation_linking': {'correct': 0.7, 'missing': 0.2, 'wrong': 0.1, 'has_triples': True} + } + ] + + avg_metrics = calculate_average_metrics(results, True) + assert isinstance(avg_metrics, dict) + assert avg_metrics['triples_evaluation_precision'] == 0.8 + assert avg_metrics['entity_presence_f1'] == 0.87 + assert avg_metrics['entity_linking_accuracy'] == 0.8 + +def test_evaluate_ontology(tmp_path): + # Create temporary test files + config = { + 'path_patterns': { + 'sys': str(tmp_path / 'outputs' / '$$onto$$_output.jsonl'), + 'gt': str(tmp_path / 'ground_truth' / '$$onto$$_gt.jsonl'), + 'onto': str(tmp_path / 'ontologies' / '$$onto$$.json'), + 'output': str(tmp_path / 'metrics' / 'ont_$$onto$$_eval_results.jsonl') + } + } + + # Create necessary directories + os.makedirs(tmp_path / 'outputs', exist_ok=True) + os.makedirs(tmp_path / 'ground_truth', exist_ok=True) + os.makedirs(tmp_path / 'ontologies', exist_ok=True) + + # Create test files + test_output = [ + { + "id": "test1", + "ents": [{"text": "John", "class": "Person"}], + "triples": [["John", "works_at", "Company"]] + } + ] + test_gt = [ + { + "id": "test1", + "sent": "John works at Company", + "triples": [{"sub": "John", "rel": "works_at", "obj": "Company"}] + } + ] + test_onto = { + "Relations": { + "works_at": [["Person", "Organization"]] + } + } + + # Write test files + write_jsonl(test_output, str(tmp_path / 'outputs' / 'test_output.jsonl')) + write_jsonl(test_gt, str(tmp_path / 'ground_truth' / 'test_gt.jsonl')) + with open(str(tmp_path / 'ontologies' / 'test.json'), 'w') as f: + json.dump(test_onto, f) + + results = evaluate_ontology('test', config) + assert isinstance(results, list) + assert len(results) > 0 + assert 'triples_evaluation' in results[0] + assert 'entity_presence' in results[0] + assert 'entity_linking' in results[0] + +def test_evaluate_results_with_config(tmp_path): + config = { + 'onto_list': ['test'], + 'path_patterns': { + 'sys': str(tmp_path / 'outputs' / '$$onto$$_output.jsonl'), + 'gt': str(tmp_path / 'ground_truth' / '$$onto$$_gt.jsonl'), + 'onto': str(tmp_path / 'ontologies' / '$$onto$$.json'), + 'output': str(tmp_path / 'metrics' / 'ont_$$onto$$_eval_results.jsonl') + }, + 'overall_output': str(tmp_path / 'overall_metrics.json') + } + + # Create test files similar to test_evaluate_ontology + os.makedirs(tmp_path / 'outputs', exist_ok=True) + os.makedirs(tmp_path / 'ground_truth', exist_ok=True) + os.makedirs(tmp_path / 'ontologies', exist_ok=True) + + test_output = [{"id": "test1", "ents": [], "triples": []}] + test_gt = [{"id": "test1", "sent": "test", "triples": []}] + test_onto = {"Relations": {}} + + write_jsonl(test_output, str(tmp_path / 'outputs' / 'test_output.jsonl')) + write_jsonl(test_gt, str(tmp_path / 'ground_truth' / 'test_gt.jsonl')) + with open(str(tmp_path / 'ontologies' / 'test.json'), 'w') as f: + json.dump(test_onto, f) + + # Test both detailed and non-detailed evaluations + from cleverswarm_python_client.libs.evaluation.detailed_eval import EvaluateResults + EvaluateResults(config, is_detailed_metrics=True) + assert os.path.exists(config['overall_output']) + + EvaluateResults(config, is_detailed_metrics=False) + assert os.path.exists(config['overall_output']) + +def test_custom_encoder(): + test_data = {"set_field": {1, 2, 3}} + encoded = json.dumps(test_data, cls=CustomEncoder) + decoded = json.loads(encoded) + assert isinstance(decoded['set_field'], list) + assert set(decoded['set_field']) == {1, 2, 3} + +def test_calculate_average_metrics_edge_cases(): + # Test with empty results + empty_results = [] + avg_metrics = calculate_average_metrics(empty_results, True) + assert isinstance(avg_metrics, dict) + + # Test with all required fields with minimal data + incomplete_results = [{ + 'triples_evaluation': { + 'triples_precision': 0.8, + 'triples_recall': 0.7, + 'triples_f1': 0.75 + }, + 'entity_presence': { + 'precision': 0.7, + 'recall': 0.8, + 'f1': 0.75, + 'has_entities': True + }, + 'entity_linking': { + 'accuracy': 0.6, + 'has_links': True + }, + 'relation_linking': { + 'correct': 0.5, + 'missing': 0.3, + 'wrong': 0.2, + 'has_triples': True + } + }] + + # Test with detailed metrics + avg_metrics = calculate_average_metrics(incomplete_results, True) + assert isinstance(avg_metrics, dict) + assert 'triples_evaluation_precision' in avg_metrics + assert 'triples_evaluation_recall' in avg_metrics + assert 'entity_linking_accuracy' in avg_metrics + + # Test with non-detailed metrics + avg_metrics = calculate_average_metrics(incomplete_results, False) + assert isinstance(avg_metrics, dict) + assert 'entity_presence_precision' in avg_metrics + assert 'triples_evaluation_precision' in avg_metrics diff --git a/tests/test_cleverswarm_client.py b/tests/test_cleverswarm_client.py new file mode 100644 index 0000000..61b4081 --- /dev/null +++ b/tests/test_cleverswarm_client.py @@ -0,0 +1,853 @@ +""" +Unit tests for cleverswarm_client module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch +import pytest +import json +from httmock import HTTMock, urlmatch + +from cleverswarm_python_client.libs.cleverswarm_client import CleverSwarmClient +from cleverswarm_python_client.libs.exceptions import ( + InvalidFileException, + JobNotFoundException, + UnauthorizedException, + InvalidUsernameOrPasswordException, + RequestErrorException, + ConnectionErrorException, + ConnectionTimeoutException, + UnexpectedConditionException, +) +from cleverswarm_python_client.libs.job_enums import JobStatus +from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI +from cleverswarm_python_client.libs.response_type_enum import ResponseTypeAPI + + +class TestCleverSwarmClient: + """Test cases for CleverSwarmClient.""" + + def setup_method(self): + """Set up test client.""" + self.client = CleverSwarmClient("http://localhost:8000/api/v0/") + + def test_init_with_base_url(self): + """Test initialization with base URL.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + assert client._base_url == "http://localhost:8000/api/v0/" + assert client._username is None + assert client._token is None + + def test_init_with_username_and_token(self): + """Test initialization with username and token.""" + client = CleverSwarmClient( + "http://localhost:8000/api/v0/", + username="testuser", + token="testtoken" + ) + assert client._base_url == "http://localhost:8000/api/v0/" + assert client._username == "testuser" + assert client._token == "testtoken" + + def test_init_with_empty_base_url(self): + """Test initialization with empty base URL.""" + client = CleverSwarmClient("") + assert client._base_url == "" + + def test_init_with_none_base_url(self): + """Test initialization with None base URL.""" + client = CleverSwarmClient(None) + assert client._base_url is None + + def test_client_initialization_attributes(self): + """Test client initialization attributes.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test all default attributes + assert client._base_url == "http://localhost:8000/api/v0/" + assert client._username is None + assert client._token is None + assert client._max_retries == 2 + assert client._timeout == 10 + assert client._triplets_template == "{}_output.jsonl" + + def test_client_initialization_with_credentials(self): + """Test client initialization with various credential combinations.""" + # Test with all parameters + client = CleverSwarmClient( + "http://localhost:8000/api/v0/", + username="testuser", + token="testtoken" + ) + assert client._username == "testuser" + assert client._token == "testtoken" + + def test_client_with_custom_attributes(self): + """Test client with custom attributes.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test that we can modify attributes + client._max_retries = 5 + client._timeout = 30 + client._triplets_template = "custom_{}_output.jsonl" + + assert client._max_retries == 5 + assert client._timeout == 30 + assert client._triplets_template == "custom_{}_output.jsonl" + + def test_client_file_operations_with_real_files(self): + """Test client file operations with real files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + + # Test that files exist and can be used + assert text_file.exists() + assert json_file.exists() + assert owl_file.exists() + + # Test that we can create a client and it recognizes the files + client = CleverSwarmClient("http://localhost:8000/api/v0/") + assert client._base_url == "http://localhost:8000/api/v0/" + + def test_file_validation_with_missing_files(self): + """Test file validation with missing files.""" + # Test with non-existent files - this should raise InvalidFileException before calling update_token + # We need to mock the update_token method to avoid the input() call + with patch.object(self.client, 'update_token'): + with pytest.raises(InvalidFileException): + self.client.create_benchmark_job( + Path("nonexistent1.txt"), + Path("nonexistent2.json"), + Path("nonexistent3.jsonl"), + True + ) + + def test_file_validation_with_existing_files(self): + """Test file validation with existing files.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create valid test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + jsonl_file = Path(temp_dir) / "test.jsonl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + jsonl_file.write_text('{"test": "data"}') + + # These should not raise exceptions + assert text_file.exists() + assert json_file.exists() + assert jsonl_file.exists() + + def test_client_state_management(self): + """Test client state management.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test initial state + assert client._token is None + assert client._username is None + + # Test setting token + client._token = "test_token" + assert client._token == "test_token" + + # Test setting username + client._username = "test_user" + assert client._username == "test_user" + + def test_client_configuration(self): + """Test client configuration options.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test default configuration + assert client._max_retries == 2 + assert client._timeout == 10 + + # Test modifying configuration + client._max_retries = 5 + client._timeout = 30 + + assert client._max_retries == 5 + assert client._timeout == 30 + + def test_triplets_template(self): + """Test triplets template functionality.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test default template + assert client._triplets_template == "{}_output.jsonl" + + # Test template formatting + formatted = client._triplets_template.format("test_job") + assert formatted == "test_job_output.jsonl" + + # Test custom template + client._triplets_template = "custom_{}_result.jsonl" + formatted = client._triplets_template.format("test_job") + assert formatted == "custom_test_job_result.jsonl" + + def test_poll_job_ready_or_failed_completed(self): + """Test polling job that is already completed.""" + with patch.object(self.client, 'get_job_status') as mock_status: + mock_status.return_value = JobStatus.Completed + + result = self.client.poll_job_ready_or_failed("job123") + + assert result == JobStatus.Completed + mock_status.assert_called_once_with("job123") + + def test_poll_job_ready_or_failed_failed(self): + """Test polling job that has failed.""" + with patch.object(self.client, 'get_job_status') as mock_status: + mock_status.return_value = JobStatus.Failed + + result = self.client.poll_job_ready_or_failed("job123") + + assert result == JobStatus.Failed + mock_status.assert_called_once_with("job123") + + def test_poll_job_ready_or_failed_processing(self): + """Test polling job that is still processing.""" + with patch.object(self.client, 'get_job_status') as mock_status: + # First call returns Processing, second call returns Completed + mock_status.side_effect = [JobStatus.Processing, JobStatus.Completed] + + with patch('time.sleep'): # Mock sleep to speed up test + result = self.client.poll_job_ready_or_failed("job123") + + assert result == JobStatus.Completed + assert mock_status.call_count == 2 + + def test_generic_http_request_executor_success(self): + """Test generic HTTP request executor with successful response.""" + def test_func(): + mock_response = Mock() + mock_response.status_code = 200 + return mock_response + + result = self.client._generic_http_request_executor(test_func) + + assert result.status_code == 200 + + def test_generic_http_request_executor_max_retries_exceeded(self): + """Test generic HTTP request executor with max retries exceeded.""" + def test_func(): + raise Exception("Persistent failure") + + with patch('time.sleep'): # Mock sleep to speed up test + with pytest.raises(Exception): + self.client._generic_http_request_executor(test_func) + + def test_job_status_enum_values(self): + """Test JobStatus enum values.""" + assert JobStatus.ReadyForProcessing == "ReadyForProcessing" + assert JobStatus.Processing == "Processing" + assert JobStatus.Completed == "Completed" + assert JobStatus.Failed == "Failed" + + def test_file_type_enum_values(self): + """Test FileTypeAPI enum values.""" + assert FileTypeAPI.AutoDetect == "AutoDetect" + assert FileTypeAPI.Markdown == "Markdown" + assert FileTypeAPI.PlainText == "PlainText" + + def test_response_type_enum_values(self): + """Test ResponseTypeAPI enum values.""" + assert ResponseTypeAPI.JsonText == "JsonText" + assert ResponseTypeAPI.JsonFile == "JsonFile" + + def test_client_with_different_base_urls(self): + """Test client with different base URL formats.""" + # Test with trailing slash + client1 = CleverSwarmClient("http://localhost:8000/api/v0/") + assert client1._base_url == "http://localhost:8000/api/v0/" + + # Test without trailing slash + client2 = CleverSwarmClient("http://localhost:8000/api/v0") + assert client2._base_url == "http://localhost:8000/api/v0" + + # Test with different port + client3 = CleverSwarmClient("http://localhost:9000/api/v1/") + assert client3._base_url == "http://localhost:9000/api/v1/" + + def test_client_initialization_edge_cases(self): + """Test client initialization with edge cases.""" + # Test with empty string + client1 = CleverSwarmClient("") + assert client1._base_url == "" + + # Test with None + client2 = CleverSwarmClient(None) + assert client2._base_url is None + + # Test with just username + client3 = CleverSwarmClient("http://localhost:8000/api/v0/", username="testuser") + assert client3._username == "testuser" + assert client3._token is None + + # Test with just token + client4 = CleverSwarmClient("http://localhost:8000/api/v0/", token="testtoken") + assert client4._token == "testtoken" + assert client4._username is None + + def test_client_attribute_modification(self): + """Test that client attributes can be modified after initialization.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Modify all attributes + client._base_url = "http://newhost:9000/api/v1/" + client._username = "newuser" + client._token = "newtoken" + client._max_retries = 10 + client._timeout = 60 + client._triplets_template = "new_{}_template.jsonl" + + # Verify changes + assert client._base_url == "http://newhost:9000/api/v1/" + assert client._username == "newuser" + assert client._token == "newtoken" + assert client._max_retries == 10 + assert client._timeout == 60 + assert client._triplets_template == "new_{}_template.jsonl" + + def test_client_string_representation(self): + """Test client string representation.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test that the client object can be converted to string + client_str = str(client) + assert isinstance(client_str, str) + assert "CleverSwarmClient" in client_str + + def test_client_equality(self): + """Test client equality comparison.""" + client1 = CleverSwarmClient("http://localhost:8000/api/v0/") + client2 = CleverSwarmClient("http://localhost:8000/api/v0/") + client3 = CleverSwarmClient("http://localhost:9000/api/v0/") + + # Different instances should not be equal + assert client1 != client2 + assert client1 != client3 + + # Same instance should be equal to itself + assert client1 == client1 + + def test_client_hash(self): + """Test client hash functionality.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test that client can be hashed + client_hash = hash(client) + assert isinstance(client_hash, int) + + # Test that same client has same hash + assert hash(client) == hash(client) + + def test_client_with_credentials_initialization(self): + """Test client initialization with various credential combinations.""" + # Test with both username and token + client1 = CleverSwarmClient( + "http://localhost:8000/api/v0/", + username="user1", + token="token1" + ) + assert client1._username == "user1" + assert client1._token == "token1" + + # Test with only username + client2 = CleverSwarmClient( + "http://localhost:8000/api/v0/", + username="user2" + ) + assert client2._username == "user2" + assert client2._token is None + + # Test with only token + client3 = CleverSwarmClient( + "http://localhost:8000/api/v0/", + token="token3" + ) + assert client3._username is None + assert client3._token == "token3" + + def test_client_template_functionality(self): + """Test client template functionality in detail.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test default template + assert client._triplets_template == "{}_output.jsonl" + + # Test template with different job IDs + job_ids = ["job1", "job2", "test_job_123", "my-special-job"] + for job_id in job_ids: + expected = f"{job_id}_output.jsonl" + actual = client._triplets_template.format(job_id) + assert actual == expected + + # Test custom template + client._triplets_template = "results_{}_final.jsonl" + for job_id in job_ids: + expected = f"results_{job_id}_final.jsonl" + actual = client._triplets_template.format(job_id) + assert actual == expected + + def test_client_retry_configuration(self): + """Test client retry configuration.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test default retry configuration + assert client._max_retries == 2 + + # Test setting different retry values + for retries in [0, 1, 5, 10]: + client._max_retries = retries + assert client._max_retries == retries + + def test_client_timeout_configuration(self): + """Test client timeout configuration.""" + client = CleverSwarmClient("http://localhost:8000/api/v0/") + + # Test default timeout + assert client._timeout == 10 + + # Test setting different timeout values + for timeout in [1, 5, 30, 60, 120]: + client._timeout = timeout + assert client._timeout == timeout + + +class TestCleverSwarmClientHTTP: + """Test cases for CleverSwarmClient HTTP operations using httmock.""" + + def setup_method(self): + """Set up test client.""" + self.client = CleverSwarmClient("http://localhost:8000/api/v0/", token="test_token") + + # Authentication Tests + @urlmatch(netloc='localhost:8000', path='/api/v0/login', method='POST') + def login_success_mock(self, url, request): + """Mock successful login response.""" + return { + 'status_code': 200, + 'content': json.dumps({ + 'access_token': 'new_token', + 'token_type': 'bearer' + }).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + @urlmatch(netloc='localhost:8000', path='/api/v0/login', method='POST') + def login_unauthorized_mock(self, url, request): + """Mock unauthorized login response.""" + return { + 'status_code': 401, + 'content': b'User provided credentials are not correct.', + 'headers': {'content-type': 'text/plain'} + } + + @urlmatch(netloc='localhost:8000', path='/api/v0/login', method='POST') + def login_invalid_credentials_mock(self, url, request): + """Mock invalid credentials login response.""" + return { + 'status_code': 400, + 'content': b'Invalid credentials', + 'headers': {'content-type': 'text/plain'} + } + + @patch('cleverswarm_python_client.libs.cleverswarm_client.getpass') + def test_update_token_success(self, mock_getpass): + """Test successful token update using httmock.""" + mock_getpass.return_value = "testpassword" + + with HTTMock(self.login_success_mock): + client = CleverSwarmClient("http://localhost:8000/api/v0/", username="testuser") + result = client.update_token() + + assert result is client # Should return self + assert client._token == "new_token" + + @patch('cleverswarm_python_client.libs.cleverswarm_client.getpass') + def test_update_token_unauthorized(self, mock_getpass): + """Test token update with unauthorized response using httmock.""" + mock_getpass.return_value = "testpassword" + + with HTTMock(self.login_unauthorized_mock): + client = CleverSwarmClient("http://localhost:8000/api/v0/", username="testuser") + + with pytest.raises(RequestErrorException, match="Failed process request, error from server"): + client.update_token() + + @patch('cleverswarm_python_client.libs.cleverswarm_client.getpass') + def test_update_token_invalid_credentials(self, mock_getpass): + """Test token update with invalid credentials using httmock.""" + mock_getpass.return_value = "testpassword" + + with HTTMock(self.login_invalid_credentials_mock): + client = CleverSwarmClient("http://localhost:8000/api/v0/", username="testuser") + + with pytest.raises(InvalidUsernameOrPasswordException): + client.update_token() + + # Job Management Tests + @urlmatch(netloc='localhost:8000', path='/api/v0/jobs', method='GET') + def jobs_list_success_mock(self, url, request): + """Mock successful jobs list response.""" + return { + 'status_code': 200, + 'content': json.dumps([ + {"id": "job1", "type": "Benchmark", "status": "Completed"}, + {"id": "job2", "type": "UnstructuredWithOntology", "status": "Processing"} + ]).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_get_jobs_list_success(self): + """Test successful job list retrieval using httmock.""" + with HTTMock(self.jobs_list_success_mock): + result = self.client.get_jobs_list(is_benchmark=True) + + assert len(result) == 1 # Only Benchmark jobs are returned + assert result[0]["id"] == "job1" + assert result[0]["type"] == "Benchmark" + + def test_get_jobs_list_filtering(self): + """Test job list retrieval with filtering using httmock.""" + with HTTMock(self.jobs_list_success_mock): + # Test benchmark jobs only + result = self.client.get_jobs_list(is_benchmark=True) + assert len(result) == 1 + assert result[0]["type"] == "Benchmark" + + # Test non-benchmark jobs only + result = self.client.get_jobs_list(is_benchmark=False) + assert len(result) == 1 + assert result[0]["type"] == "UnstructuredWithOntology" + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='GET') + def job_status_success_mock(self, url, request): + """Mock successful job status response.""" + return { + 'status_code': 200, + 'content': json.dumps("Completed").encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='GET') + def job_status_not_found_mock(self, url, request): + """Mock job not found response.""" + return { + 'status_code': 404, + 'content': b'Job not found', + 'headers': {'content-type': 'text/plain'} + } + + def test_get_job_status_success(self): + """Test successful job status retrieval using httmock.""" + with HTTMock(self.job_status_success_mock): + result = self.client.get_job_status("job123") + + assert result == JobStatus.Completed + + def test_get_job_status_not_found(self): + """Test job status retrieval for non-existent job using httmock.""" + with HTTMock(self.job_status_not_found_mock): + with pytest.raises(JobNotFoundException): + self.client.get_job_status("nonexistent") + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='GET') + def job_details_success_mock(self, url, request): + """Mock successful job details response.""" + return { + 'status_code': 200, + 'content': json.dumps({ + "id": "job123", + "status": "Completed", + "type": "Benchmark", + "created": "2023-01-01T00:00:00Z" + }).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_get_job_details_success(self): + """Test successful job details retrieval using httmock.""" + with HTTMock(self.job_details_success_mock): + result = self.client.get_job_details("job123") + + assert result["id"] == "job123" + assert result["status"] == "Completed" + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='DELETE') + def delete_job_success_mock(self, url, request): + """Mock successful job deletion response.""" + return { + 'status_code': 200, + 'content': b'Job deleted', + 'headers': {'content-type': 'text/plain'} + } + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='DELETE') + def delete_job_not_found_mock(self, url, request): + """Mock job not found for deletion response.""" + return { + 'status_code': 404, + 'content': b'Job not found', + 'headers': {'content-type': 'text/plain'} + } + + def test_delete_job_success(self): + """Test successful job deletion using httmock.""" + with HTTMock(self.delete_job_success_mock): + result = self.client.delete_job("job123") + + assert result is True + + def test_delete_job_not_found(self): + """Test job deletion for non-existent job using httmock.""" + with HTTMock(self.delete_job_not_found_mock): + with pytest.raises(JobNotFoundException): + self.client.delete_job("nonexistent") + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/jobs/.*', method='PUT') + def retry_job_success_mock(self, url, request): + """Mock successful job retry response.""" + return { + 'status_code': 200, + 'content': json.dumps("Processing").encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_retry_job_success(self): + """Test successful job retry using httmock.""" + with HTTMock(self.retry_job_success_mock): + result = self.client.retry_job("job123") + + assert result == JobStatus.Processing + + # File Upload Tests + @urlmatch(netloc='localhost:8000', path='/api/v0/benchmark', method='POST') + def benchmark_job_success_mock(self, url, request): + """Mock successful benchmark job creation response.""" + return { + 'status_code': 200, + 'content': json.dumps({"job_id": "benchmark_job_123"}).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_create_benchmark_job_success(self): + """Test successful benchmark job creation using httmock.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create valid test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + jsonl_file = Path(temp_dir) / "test.jsonl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + jsonl_file.write_text('{"test": "data"}') + + with HTTMock(self.benchmark_job_success_mock): + with patch.object(self.client, 'update_token'): # Mock update_token to avoid input() + result = self.client.create_benchmark_job( + text_file, json_file, jsonl_file, True + ) + + assert result == "benchmark_job_123" + + @urlmatch(netloc='localhost:8000', path='/api/v0/unstructured/with_ontology', method='POST') + def unstructured_job_success_mock(self, url, request): + """Mock successful unstructured to KG job creation response.""" + return { + 'status_code': 201, + 'content': json.dumps({"job_id": "unstructured_job_123"}).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_create_unstructured_to_kg_job_success(self): + """Test successful unstructured to KG job creation using httmock.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create valid test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + + with HTTMock(self.unstructured_job_success_mock): + with patch.object(self.client, 'update_token'): # Mock update_token to avoid input() + result = self.client.create_unstructured_to_kg_job( + text_file, json_file, owl_file + ) + + assert result == "unstructured_job_123" + + @urlmatch(netloc='localhost:8000', path='/api/v0/unstructured/with_wildcards', method='POST') + def unstructured_wildcards_job_success_mock(self, url, request): + """Mock successful unstructured to KG wildcards job creation response.""" + return { + 'status_code': 201, + 'content': json.dumps({"job_id": "wildcards_job_123"}).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + def test_create_unstructured_to_kg_wildcards_job_success(self): + """Test successful unstructured to KG wildcards job creation using httmock.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create valid test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + wildcards_file = Path(temp_dir) / "test.wildcards" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + wildcards_file.write_text("wildcard1\nwildcard2") + + with HTTMock(self.unstructured_wildcards_job_success_mock): + with patch.object(self.client, 'update_token'): # Mock update_token to avoid input() + result = self.client.create_unstructured_to_kg_wildcards_job( + text_file, json_file, owl_file, wildcards_file + ) + + assert result == "wildcards_job_123" + + # File Download Tests + @urlmatch(netloc='localhost:8000', path=r'/api/v0/benchmark/.*', method='GET') + def benchmark_file_download_mock(self, url, request): + """Mock successful benchmark file download response.""" + return { + 'status_code': 200, + 'content': b'{"test": "benchmark_data"}', + 'headers': {'content-type': 'application/json'} + } + + def test_retrieve_benchmark_files_success(self): + """Test successful benchmark files retrieval using httmock.""" + with tempfile.TemporaryDirectory() as temp_dir: + output_file = Path(temp_dir) / "output.jsonl" + + with HTTMock(self.benchmark_file_download_mock): + with patch.object(self.client, 'update_token'): # Mock update_token to avoid input() + result = self.client.retrieve_benchmark_files("job123", [output_file]) + + assert result == [output_file] + + @urlmatch(netloc='localhost:8000', path=r'/api/v0/unstructured/.*', method='GET') + def unstructured_file_download_mock(self, url, request): + """Mock successful unstructured file download response.""" + return { + 'status_code': 200, + 'content': b'{"test": "unstructured_data"}', + 'headers': {'content-type': 'application/json'} + } + + def test_retrieve_unstructured_to_kg_files_success(self): + """Test successful unstructured to KG files retrieval using httmock.""" + with tempfile.TemporaryDirectory() as temp_dir: + output_file = Path(temp_dir) / "output.jsonl" + + with HTTMock(self.unstructured_file_download_mock): + with patch.object(self.client, 'update_token'): # Mock update_token to avoid input() + result = self.client.retrieve_unstructured_to_kg_files("job123", output_file) + + assert result == output_file + + # Error Handling Tests + @urlmatch(netloc='localhost:8000', path='/api/v0/test', method='GET') + def generic_http_success_mock(self, url, request): + """Mock successful generic HTTP response.""" + return { + 'status_code': 200, + 'content': json.dumps({"result": "success"}).encode('utf-8'), + 'headers': {'content-type': 'application/json'} + } + + @urlmatch(netloc='localhost:8000', path='/api/v0/test', method='GET') + def generic_http_error_mock(self, url, request): + """Mock error generic HTTP response.""" + return { + 'status_code': 500, + 'content': b'Internal Server Error', + 'headers': {'content-type': 'text/plain'} + } + + def test_generic_http_request_executor_success(self): + """Test generic HTTP request executor with successful response using httmock.""" + with HTTMock(self.generic_http_success_mock): + def test_func(): + import requests + return requests.get("http://localhost:8000/api/v0/test") + + result = self.client._generic_http_request_executor(test_func) + + assert result.status_code == 200 + + # Connection Error Tests + def test_generic_http_request_executor_connection_error(self): + """Test generic HTTP request executor with connection error.""" + def test_func(): + import requests + raise requests.exceptions.ConnectionError("Connection failed") + + with pytest.raises(ConnectionErrorException): + self.client._generic_http_request_executor(test_func) + + def test_generic_http_request_executor_read_timeout_error(self): + """Test generic HTTP request executor with read timeout error.""" + def test_func(): + import requests + raise requests.exceptions.ReadTimeout("Read timeout") + + with pytest.raises(ConnectionTimeoutException): + self.client._generic_http_request_executor(test_func) + + def test_generic_http_request_executor_http_error(self): + """Test generic HTTP request executor with HTTP error.""" + def test_func(): + import requests + raise requests.exceptions.HTTPError("HTTP error") + + with pytest.raises(RequestErrorException): + self.client._generic_http_request_executor(test_func) + + def test_generic_http_request_executor_request_error(self): + """Test generic HTTP request executor with request error.""" + def test_func(): + import requests + raise requests.exceptions.RequestException("Request error") + + with pytest.raises(UnexpectedConditionException, match="Client error while processing authorization"): + self.client._generic_http_request_executor(test_func) + + def test_generic_http_request_executor_unexpected_error(self): + """Test generic HTTP request executor with unexpected error.""" + def test_func(): + raise ValueError("Unexpected error") + + with pytest.raises(UnexpectedConditionException): + self.client._generic_http_request_executor(test_func) \ No newline at end of file diff --git a/tests/test_cswarm_benchmark_client.py b/tests/test_cswarm_benchmark_client.py new file mode 100644 index 0000000..f6312b2 --- /dev/null +++ b/tests/test_cswarm_benchmark_client.py @@ -0,0 +1,587 @@ +""" +Unit tests for cswarm_benchmark_client module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock +import pytest + +from cleverswarm_python_client.cswarm_benchmark_client import BenchmarkCLI, main +from cleverswarm_python_client.libs.exceptions import ClientException, JobNotFoundException +from cleverswarm_python_client.libs.job_enums import JobStatus + + +class TestBenchmarkCLIInit: + """Test cases for BenchmarkCLI initialization.""" + + def test_init_with_defaults(self): + """Test initialization with default parameters.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI(temp_dir) + + assert cli._base_url == "http://localhost:8000/api/v0/" + assert cli._base_path == Path(temp_dir).resolve() + assert cli._detailed_metrics is False + assert cli._client is not None + + def test_init_with_custom_params(self): + """Test initialization with custom parameters.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI( + temp_dir, + base_url="http://test:8080/api/", + detailed_metrics=True, + username="testuser", + token="testtoken" + ) + + assert cli._base_url == "http://test:8080/api/" + assert cli._base_path == Path(temp_dir).resolve() + assert cli._detailed_metrics is True + assert cli._client is not None + + def test_init_ontologies_mapping(self): + """Test that ontologies mapping is correctly initialized.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI(temp_dir) + + assert len(cli._all_ontologies) == 21 + assert cli._all_ontologies[1] == "1_university" + assert cli._all_ontologies[21] == "21_complex" + + def test_init_path_templates(self): + """Test that path templates are correctly initialized.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI(temp_dir) + + assert cli._benchmark_path == "benchmark_data" + assert cli._input_template == "test_new/ont_$$onto$$_test.jsonl" + assert cli._onto_template == "ontologies_enriched/$$onto$$_ontology_with_descriptions.json" + assert cli._ground_truth_template == "ground_truth_new/ont_$$onto$$_ground_truth.jsonl" + assert cli._output_template == "benchmark_data/outputs_new/$$onto$$_output.jsonl" + + +class TestBenchmarkCLIOntologyValidation: + """Test cases for BenchmarkCLI ontology validation methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + def test_find_invalid_ontologies_empty_list(self): + """Test finding invalid ontologies with empty list.""" + result = self.cli.find_invalid_ontologies([]) + assert result == [] + + def test_find_invalid_ontologies_all_valid(self): + """Test finding invalid ontologies with all valid IDs.""" + result = self.cli.find_invalid_ontologies([1, 5, 10, 21]) + assert result == [] + + def test_find_invalid_ontologies_some_invalid(self): + """Test finding invalid ontologies with some invalid IDs.""" + result = self.cli.find_invalid_ontologies([1, 5, 99, 10, 100]) + assert result == [99, 100] + + def test_find_invalid_ontologies_all_invalid(self): + """Test finding invalid ontologies with all invalid IDs.""" + result = self.cli.find_invalid_ontologies([99, 100, 101]) + assert result == [99, 100, 101] + + def test_find_invalid_ontologies_boundary_values(self): + """Test finding invalid ontologies with boundary values.""" + # Test valid boundary values + result = self.cli.find_invalid_ontologies([1, 21]) + assert result == [] + + # Test invalid boundary values + result = self.cli.find_invalid_ontologies([0, 22]) + assert result == [0, 22] + + +class TestBenchmarkCLIJobCreation: + """Test cases for BenchmarkCLI job creation methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + def test_create_benchmark_empty_ontologies(self): + """Test creating benchmark with empty ontologies list.""" + result = self.cli.create_benchmark([]) + assert result is None + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_create_benchmark_single_ontology(self, mock_client_class): + """Test creating benchmark with single ontology.""" + mock_client = Mock() + mock_client.create_benchmark_job.return_value = "test_job_123" + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with tempfile.TemporaryDirectory() as temp_dir: + # Create test files + test_file = Path(temp_dir) / "test_new" / "ont_1_university_test.jsonl" + test_file.parent.mkdir(parents=True) + test_file.write_text('{"test": "data"}') + + onto_file = Path(temp_dir) / "ontologies_enriched" / "1_university_ontology_with_descriptions.json" + onto_file.parent.mkdir(parents=True) + onto_file.write_text('{"ontology": "data"}') + + gt_file = Path(temp_dir) / "ground_truth_new" / "ont_1_university_ground_truth.jsonl" + gt_file.parent.mkdir(parents=True) + gt_file.write_text('{"ground_truth": "data"}') + + # Update base path + self.cli._base_path = Path(temp_dir) + + result = self.cli.create_benchmark([1]) + assert result == "test_job_123" + mock_client.create_benchmark_job.assert_called_once() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_create_benchmark_multiple_ontologies(self, mock_client_class): + """Test creating benchmark with multiple ontologies.""" + mock_client = Mock() + mock_client.create_benchmark_job.return_value = "test_job_123" + mock_client.append_to_benchmark_job.return_value = True + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with tempfile.TemporaryDirectory() as temp_dir: + # Create test files for multiple ontologies + for onto_id in [1, 2]: + onto_name = self.cli._all_ontologies[onto_id] + + test_file = Path(temp_dir) / "input_data" / "test_new" / f"ont_{onto_name}_test.jsonl" + test_file.parent.mkdir(parents=True, exist_ok=True) + test_file.write_text('{"test": "data"}') + + onto_file = Path(temp_dir) / "input_data" / "ontologies_enriched" / f"{onto_name}_ontology_with_descriptions.json" + onto_file.parent.mkdir(parents=True, exist_ok=True) + onto_file.write_text('{"ontology": "data"}') + + gt_file = Path(temp_dir) / "input_data" / "ground_truth_new" / f"ont_{onto_name}_ground_truth.jsonl" + gt_file.parent.mkdir(parents=True, exist_ok=True) + gt_file.write_text('{"ground_truth": "data"}') + + # Update base path + self.cli._base_path = Path(temp_dir) + + result = self.cli.create_benchmark([1, 2]) + assert result == "test_job_123" + assert mock_client.create_benchmark_job.call_count == 1 + assert mock_client.append_to_benchmark_job.call_count == 1 + + +class TestBenchmarkCLIJobPolling: + """Test cases for BenchmarkCLI job polling methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_success(self, mock_client_class): + """Test successful job polling.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Completed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_failed(self, mock_client_class): + """Test job polling with failed job.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Failed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with pytest.raises(ClientException, match="finished without completing"): + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_processing(self, mock_client_class): + """Test job polling with processing job.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_ready_for_processing(self, mock_client_class): + """Test job polling with ready for processing job.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.ReadyForProcessing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_invalid_status(self, mock_client_class): + """Test job polling with invalid status.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Created + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with pytest.raises(ClientException, match="is not processing"): + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_poll_job_to_completion_poll_failed(self, mock_client_class): + """Test job polling where poll returns failed.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Failed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with pytest.raises(ClientException, match="finished without completing"): + self.cli.poll_job_to_completion("test_job_123") + + +class TestBenchmarkCLIJobDownload: + """Test cases for BenchmarkCLI job download methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_download_job_results_success(self, mock_client_class): + """Test successful job results download.""" + mock_client = Mock() + mock_client.retrieve_benchmark_files.return_value = True + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + # Should not raise any exception + self.cli.download_job_results("test_job_123", [1, 2]) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_download_job_results_creates_output_dirs(self, mock_client_class): + """Test that download creates necessary output directories.""" + mock_client = Mock() + mock_client.retrieve_benchmark_files.return_value = True + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + self.cli.download_job_results("test_job_123", [1]) + + # Check that retrieve_benchmark_files was called with correct paths + mock_client.retrieve_benchmark_files.assert_called_once() + call_args = mock_client.retrieve_benchmark_files.call_args + assert call_args[0][0] == "test_job_123" # job_id + output_files = call_args[0][1] # output_filenames + assert len(output_files) == 1 + assert "university_output.jsonl" in str(output_files[0]) + + +class TestBenchmarkCLIJobManagement: + """Test cases for BenchmarkCLI job management methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_list_server_jobs_success(self, mock_client_class): + """Test successful server jobs listing.""" + mock_client = Mock() + mock_jobs = [ + {"id": "job1", "status": "Completed"}, + {"id": "job2", "status": "Processing"} + ] + mock_client.get_jobs_list.return_value = mock_jobs + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + result = self.cli.list_server_jobs() + assert result == mock_jobs + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_extract_server_job_ontologies_success(self, mock_client_class): + """Test successful extraction of server job ontologies.""" + mock_client = Mock() + mock_job_details = { + "ontology_sources": ["1_university_ontology.json", "2_musicalwork_ontology.json"] + } + mock_client.get_job_details.return_value = mock_job_details + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + result = self.cli.extract_server_job_ontologies("test_job_123") + assert result == [1, 2] + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_extract_server_job_ontologies_not_found(self, mock_client_class): + """Test extraction of ontologies from non-existent job.""" + mock_client = Mock() + mock_client.get_job_details.return_value = None + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with pytest.raises(JobNotFoundException): + self.cli.extract_server_job_ontologies("nonexistent_job") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_delete_server_job_success(self, mock_client_class): + """Test successful server job deletion.""" + mock_client = Mock() + mock_client.delete_job.return_value = True + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception + self.cli.delete_server_job("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_delete_server_job_failure(self, mock_client_class): + """Test server job deletion failure.""" + mock_client = Mock() + mock_client.delete_job.return_value = False + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception, but should handle failure gracefully + self.cli.delete_server_job("test_job_123") + + +class TestBenchmarkCLIEvaluation: + """Test cases for BenchmarkCLI evaluation methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.EvaluateResults') + def test_evaluate_results_locally_success(self, mock_evaluate_results): + """Test successful local results evaluation.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + result = self.cli.evaluate_results_locally([1, 2]) + + assert result == self.cli._base_path / "benchmark_data" / "local_metrics" + mock_evaluate_results.assert_called_once() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.EvaluateResults') + def test_evaluate_results_locally_detailed_metrics(self, mock_evaluate_results): + """Test local results evaluation with detailed metrics.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + self.cli._detailed_metrics = True + + result = self.cli.evaluate_results_locally([1, 2]) + + assert result == self.cli._base_path / "benchmark_data" / "local_metrics" + # Check that detailed metrics flag is passed + call_args = mock_evaluate_results.call_args + assert call_args[0][1] is True # is_detailed_metrics parameter + + +class TestBenchmarkCLIUtilityMethods: + """Test cases for BenchmarkCLI utility methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + def test_print_ontologies_ids(self, capsys): + """Test printing ontology IDs.""" + self.cli.print_ontologies_ids() + + captured = capsys.readouterr() + assert "ID, Name" in captured.out + assert "1, 1_university" in captured.out + assert "21, 21_complex" in captured.out + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_print_server_jobs_basic(self, mock_client_class, capsys): + """Test printing server jobs in basic mode.""" + mock_client = Mock() + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "Benchmark", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL" + } + ] + mock_client.get_jobs_list.return_value = mock_jobs + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + self.cli.print_server_jobs() + + captured = capsys.readouterr() + assert "Job ID: job1" in captured.out + assert "Status: Completed" in captured.out + + @patch('cleverswarm_python_client.cswarm_benchmark_client.CleverSwarmClient') + def test_print_server_jobs_detailed(self, mock_client_class, capsys): + """Test printing server jobs in detailed mode.""" + mock_client = Mock() + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "Benchmark", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ground_truth_sources": ["gt.jsonl"] + } + ] + mock_client.get_jobs_list.return_value = mock_jobs + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + self.cli.print_server_jobs(detailed=True) + + captured = capsys.readouterr() + assert "Job ID: job1" in captured.out + assert "Unstructured text input files:" in captured.out + assert "Ontology input files:" in captured.out + assert "Ground-truth input files:" in captured.out + + +class TestBenchmarkCLIMainFunction: + """Test cases for BenchmarkCLI main function.""" + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_list_ontologies(self, mock_cli_class): + """Test main function with list ontologies action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'list-ontologies-ids-and-exit']): + main() + + mock_cli.print_ontologies_ids.assert_called_once() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_create_and_exit(self, mock_cli_class): + """Test main function with create and exit action.""" + mock_cli = Mock() + mock_cli.create_benchmark.return_value = "test_job_123" + mock_cli.find_invalid_ontologies.return_value = [] + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-and-exit', '--ontologies_ids', '1', '2']): + main() # Should not raise SystemExit on success + + mock_cli.create_benchmark.assert_called_once_with([1, 2]) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_invalid_ontologies(self, mock_cli_class): + """Test main function with invalid ontology IDs.""" + mock_cli = Mock() + mock_cli.find_invalid_ontologies.return_value = [99, 100] + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-and-exit', '--ontologies_ids', '1', '99', '100']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_client_exception(self, mock_cli_class): + """Test main function with client exception.""" + mock_cli = Mock() + mock_cli.find_invalid_ontologies.side_effect = ClientException("Test error") + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-and-exit', '--ontologies_ids', '1']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_unexpected_exception(self, mock_cli_class): + """Test main function with unexpected exception.""" + mock_cli = Mock() + mock_cli.find_invalid_ontologies.side_effect = Exception("Unexpected error") + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-and-exit', '--ontologies_ids', '1']): + with pytest.raises(SystemExit): + main() + + +class TestBenchmarkCLIIntegration: + """Integration test cases for BenchmarkCLI.""" + + def test_full_workflow_simulation(self): + """Test a complete workflow simulation.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI(temp_dir) + + # Test ontology validation + invalid_ids = cli.find_invalid_ontologies([1, 2, 99]) + assert invalid_ids == [99] + + # Test valid ontologies + valid_ids = cli.find_invalid_ontologies([1, 2, 3]) + assert valid_ids == [] + + # Test printing ontologies + with patch('builtins.print') as mock_print: + cli.print_ontologies_ids() + assert mock_print.call_count > 0 + + def test_path_template_substitution(self): + """Test path template substitution.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = BenchmarkCLI(temp_dir) + + # Test template substitution + onto_name = cli._all_ontologies[1] + input_path = cli._input_template.replace("$$onto$$", onto_name) + assert input_path == f"test_new/ont_{onto_name}_test.jsonl" + + onto_path = cli._onto_template.replace("$$onto$$", onto_name) + assert onto_path == f"ontologies_enriched/{onto_name}_ontology_with_descriptions.json" diff --git a/tests/test_cswarm_benchmark_client_extended.py b/tests/test_cswarm_benchmark_client_extended.py new file mode 100644 index 0000000..c035bed --- /dev/null +++ b/tests/test_cswarm_benchmark_client_extended.py @@ -0,0 +1,351 @@ +""" +Extended unit tests for cswarm_benchmark_client module to achieve higher coverage. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch +import pytest + +from cleverswarm_python_client.cswarm_benchmark_client import BenchmarkCLI, main +from cleverswarm_python_client.libs.exceptions import ClientException + + +class TestBenchmarkCLIExtended: + """Extended test cases for BenchmarkCLI to achieve higher coverage.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = BenchmarkCLI(temp_dir) + + def test_evaluate_results_locally_with_valid_ontologies(self): + """Test local evaluation with valid ontology IDs.""" + with patch.object(self.cli, 'find_invalid_ontologies') as mock_find: + mock_find.return_value = [] + with patch('cleverswarm_python_client.cswarm_benchmark_client.EvaluateResults') as mock_eval: + mock_eval.return_value = {"results": "test"} + result = self.cli.evaluate_results_locally([1, 2]) + assert result is not None + + def test_print_server_jobs_with_empty_list(self): + """Test printing server jobs with empty list.""" + with patch.object(self.cli, 'list_server_jobs') as mock_list: + mock_list.return_value = [] + self.cli.print_server_jobs() + # Should not raise any exception + + def test_print_server_jobs_detailed_with_wildcards(self): + """Test printing server jobs with detailed output including wildcards.""" + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithWildcards", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"], + "wildcards_sources": ["test.wildcards"], + "ground_truth_sources": ["test_gt.jsonl"] + } + ] + + with patch.object(self.cli, 'list_server_jobs') as mock_list: + mock_list.return_value = mock_jobs + self.cli.print_server_jobs(detailed=True) + + def test_print_server_jobs_detailed_without_wildcards(self): + """Test printing server jobs with detailed output without wildcards.""" + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"], + "ground_truth_sources": ["test_gt.jsonl"] + } + ] + + with patch.object(self.cli, 'list_server_jobs') as mock_list: + mock_list.return_value = mock_jobs + self.cli.print_server_jobs(detailed=True) + + def test_create_benchmark_with_single_ontology(self): + """Test creating benchmark with single ontology.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.create_benchmark_job.return_value = "test_job_123" + mock_client.append_to_benchmark_job.return_value = True + + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + # Create test files + test_file = Path(temp_dir) / "input_data" / "test_new" / "ont_university_test.jsonl" + test_file.parent.mkdir(parents=True, exist_ok=True) + test_file.write_text('{"test": "data"}') + + onto_file = Path(temp_dir) / "input_data" / "ontologies_enriched" / "university_ontology_with_descriptions.json" + onto_file.parent.mkdir(parents=True, exist_ok=True) + onto_file.write_text('{"ontology": "data"}') + + gt_file = Path(temp_dir) / "input_data" / "ground_truth_new" / "ont_university_ground_truth.jsonl" + gt_file.parent.mkdir(parents=True, exist_ok=True) + gt_file.write_text('{"ground_truth": "data"}') + + result = self.cli.create_benchmark([1]) + assert result == "test_job_123" + + def test_create_benchmark_with_multiple_ontologies(self): + """Test creating benchmark with multiple ontologies.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.create_benchmark_job.return_value = "test_job_123" + mock_client.append_to_benchmark_job.return_value = True + + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + # Create test files for multiple ontologies + for onto_id in [1, 2]: + onto_name = self.cli._all_ontologies[onto_id] + + test_file = Path(temp_dir) / "input_data" / "test_new" / f"ont_{onto_name}_test.jsonl" + test_file.parent.mkdir(parents=True, exist_ok=True) + test_file.write_text('{"test": "data"}') + + onto_file = Path(temp_dir) / "input_data" / "ontologies_enriched" / f"{onto_name}_ontology_with_descriptions.json" + onto_file.parent.mkdir(parents=True, exist_ok=True) + onto_file.write_text('{"ontology": "data"}') + + gt_file = Path(temp_dir) / "input_data" / "ground_truth_new" / f"ont_{onto_name}_ground_truth.jsonl" + gt_file.parent.mkdir(parents=True, exist_ok=True) + gt_file.write_text('{"ground_truth": "data"}') + + result = self.cli.create_benchmark([1, 2]) + assert result == "test_job_123" + + def test_download_job_results_with_multiple_ontologies(self): + """Test downloading job results with multiple ontologies.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.retrieve_benchmark_files.return_value = True + + with tempfile.TemporaryDirectory() as temp_dir: + self.cli._base_path = Path(temp_dir) + + self.cli.download_job_results("test_job_123", [1, 2]) + + # Verify that retrieve_benchmark_files was called + mock_client.retrieve_benchmark_files.assert_called_once() + + + +class TestBenchmarkCLIMainFunctionExtended: + """Extended test cases for BenchmarkCLI main function to achieve higher coverage.""" + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_delete_and_exit(self, mock_cli_class): + """Test main function with delete and exit action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'delete-and-exit', '--job_id', 'test_job_123']): + main() + + mock_cli.delete_server_job.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_delete_and_exit_missing_job_id(self, mock_cli_class): + """Test main function with delete and exit action missing job_id.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'delete-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_metrics_and_exit(self, mock_cli_class): + """Test main function with metrics and exit action.""" + mock_cli = Mock() + mock_cli.evaluate_results_locally.return_value = Path("results") + mock_cli.find_invalid_ontologies.return_value = [] + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'metrics-and-exit', '--local_metrics_ontologies_ids', '1', '2']): + main() + + mock_cli.evaluate_results_locally.assert_called_once_with([1, 2]) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_metrics_and_exit_missing_ontologies(self, mock_cli_class): + """Test main function with metrics and exit action missing ontologies.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'metrics-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_download_metrics_and_exit(self, mock_cli_class): + """Test main function with download-metrics-and-exit action.""" + mock_cli = Mock() + mock_cli.poll_job_to_completion.return_value = None + mock_cli.extract_server_job_ontologies.return_value = [1, 2] + mock_cli.download_job_results.return_value = None + mock_cli.evaluate_results_locally.return_value = Path("results") + mock_cli.find_invalid_ontologies.return_value = [] + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'download-metrics-and-exit', '--job_id', 'test_job_123']): + main() + + mock_cli.poll_job_to_completion.assert_called_once_with("test_job_123") + # Note: extract_server_job_ontologies is called twice in the actual code + assert mock_cli.extract_server_job_ontologies.call_count == 2 + mock_cli.download_job_results.assert_called_once_with("test_job_123", [1, 2]) + mock_cli.evaluate_results_locally.assert_called_once_with([1, 2]) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_download_metrics_and_exit_missing_job_id(self, mock_cli_class): + """Test main function with download-metrics-and-exit action missing job_id.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'download-metrics-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_list_benchmark_jobs_and_exit(self, mock_cli_class): + """Test main function with list-benchmark-jobs-and-exit action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'list-benchmark-jobs-and-exit']): + main() + + mock_cli.print_server_jobs.assert_called_once_with(False) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_create_download_metrics_delete(self, mock_cli_class): + """Test main function with create-download-metrics-delete action.""" + mock_cli = Mock() + mock_cli.find_invalid_ontologies.return_value = [] + mock_cli.create_benchmark.return_value = "test_job_123" + mock_cli.poll_job_to_completion.return_value = None + mock_cli.extract_server_job_ontologies.return_value = [1, 2] + mock_cli.download_job_results.return_value = None + mock_cli.evaluate_results_locally.return_value = Path("results") + mock_cli.delete_server_job.return_value = None + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-download-metrics-delete', '--ontologies_ids', '1', '2']): + main() + + mock_cli.create_benchmark.assert_called_once_with([1, 2]) + mock_cli.poll_job_to_completion.assert_called_once_with("test_job_123") + mock_cli.download_job_results.assert_called_once_with("test_job_123", [1, 2]) + mock_cli.evaluate_results_locally.assert_called_once_with([1, 2]) + mock_cli.delete_server_job.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_create_download_metrics_delete_missing_ontologies(self, mock_cli_class): + """Test main function with create-download-metrics-delete action missing ontologies.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-download-metrics-delete']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_create_download_metrics(self, mock_cli_class): + """Test main function with create-download-metrics action.""" + mock_cli = Mock() + mock_cli.find_invalid_ontologies.return_value = [] + mock_cli.create_benchmark.return_value = "test_job_123" + mock_cli.poll_job_to_completion.return_value = None + mock_cli.extract_server_job_ontologies.return_value = [1, 2] + mock_cli.download_job_results.return_value = None + mock_cli.evaluate_results_locally.return_value = Path("results") + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-download-metrics', '--ontologies_ids', '1', '2']): + main() + + mock_cli.create_benchmark.assert_called_once_with([1, 2]) + mock_cli.poll_job_to_completion.assert_called_once_with("test_job_123") + mock_cli.download_job_results.assert_called_once_with("test_job_123", [1, 2]) + mock_cli.evaluate_results_locally.assert_called_once_with([1, 2]) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_create_download_metrics_missing_ontologies(self, mock_cli_class): + """Test main function with create-download-metrics action missing ontologies.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'create-download-metrics']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_with_detailed_flag(self, mock_cli_class): + """Test main function with detailed flag.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'list-benchmark-jobs-and-exit', '--detailed']): + main() + + mock_cli.print_server_jobs.assert_called_once_with(True) + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_with_custom_server_url(self, mock_cli_class): + """Test main function with custom server URL.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'list-ontologies-ids-and-exit', '--server_url', 'http://custom:8000/api/']): + main() + + mock_cli_class.assert_called_once() + call_args = mock_cli_class.call_args + # Check that the server URL was passed as the second positional argument + assert call_args[0][1] == 'http://custom:8000/api/' + + @patch('cleverswarm_python_client.cswarm_benchmark_client.BenchmarkCLI') + def test_main_with_username_and_token(self, mock_cli_class): + """Test main function with username and token.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_benchmark_client.py', '--action', 'list-ontologies-ids-and-exit', '--username', 'testuser', '--token', 'testtoken']): + main() + + mock_cli_class.assert_called_once() + call_args = mock_cli_class.call_args + assert call_args.kwargs['username'] == 'testuser' + assert call_args.kwargs['token'] == 'testtoken' diff --git a/tests/test_cswarm_text_to_kg_client.py b/tests/test_cswarm_text_to_kg_client.py new file mode 100644 index 0000000..a7b0b1f --- /dev/null +++ b/tests/test_cswarm_text_to_kg_client.py @@ -0,0 +1,742 @@ +""" +Unit tests for cswarm_text_to_kg_client module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock +import pytest + +from cleverswarm_python_client.cswarm_text_to_kg_client import TextoToKGCLI, main +from cleverswarm_python_client.libs.exceptions import ClientException +from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI +from cleverswarm_python_client.libs.job_enums import JobStatus, JobType + + +class TestTextoToKGCLIInit: + """Test cases for TextoToKGCLI initialization.""" + + def test_init_with_defaults(self): + """Test initialization with default parameters.""" + cli = TextoToKGCLI("http://localhost:8000/api/v0/") + + assert cli._detailed is False + assert cli._input_prefix == Path("../..").resolve() + assert cli._output_prefix == Path("../..").resolve() + assert cli._client is not None + + def test_init_with_custom_params(self): + """Test initialization with custom parameters.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = TextoToKGCLI( + "http://test:8080/api/", + detailed=True, + input_prefix=temp_dir, + output_prefix=temp_dir, + username="testuser", + token="testtoken" + ) + + assert cli._detailed is True + assert cli._input_prefix == Path(temp_dir).resolve() + assert cli._output_prefix == Path(temp_dir).resolve() + assert cli._client is not None + + def test_init_with_none_prefixes(self): + """Test initialization with None prefixes.""" + cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=None, + output_prefix=None + ) + + assert cli._input_prefix == Path("../..").resolve() + assert cli._output_prefix == Path("../..").resolve() + + def test_init_invalid_input_prefix(self): + """Test initialization with invalid input prefix.""" + with pytest.raises(ClientException, match="Input prefix does not exist"): + TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix="/nonexistent/path" + ) + + def test_init_creates_output_prefix(self): + """Test that initialization creates output prefix if it doesn't exist.""" + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = Path(temp_dir) / "new_output_dir" + + cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=str(output_dir) + ) + + assert output_dir.exists() + assert output_dir.is_dir() + + def test_init_output_prefix_creation_failure(self): + """Test initialization when output prefix creation fails.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create a file with the same name as the output directory + output_file = Path(temp_dir) / "output_file" + output_file.write_text("test") + + # This test expects a FileExistsError when trying to create a directory + # where a file with the same name already exists + with pytest.raises(FileExistsError): + TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=str(output_file) + ) + + +class TestTextoToKGCLIJobCreation: + """Test cases for TextoToKGCLI job creation methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + def test_create_job_success(self): + """Test successful job creation.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file: + owl_file.write('test') + owl_file.flush() + + with patch.object(self.cli._client, 'create_unstructured_to_kg_job') as mock_create: + mock_create.return_value = "test_job_123" + + result = self.cli.create_job( + text_file.name, + json_file.name, + owl_file.name + ) + + assert result == "test_job_123" + mock_create.assert_called_once() + + def test_create_job_with_wildcards(self): + """Test job creation with wildcards.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file: + owl_file.write('test') + owl_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.wildcards') as wildcards_file: + wildcards_file.write('{"wildcards": "data"}') + wildcards_file.flush() + + with patch.object(self.cli._client, 'create_unstructured_to_kg_wildcards_job') as mock_create: + mock_create.return_value = "test_job_123" + + result = self.cli.create_job( + text_file.name, + json_file.name, + owl_file.name, + wildcards_file.name + ) + + assert result == "test_job_123" + mock_create.assert_called_once() + + def test_create_job_with_force_filetype(self): + """Test job creation with forced file type.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file: + owl_file.write('test') + owl_file.flush() + + with patch.object(self.cli._client, 'create_unstructured_to_kg_job') as mock_create: + mock_create.return_value = "test_job_123" + + result = self.cli.create_job( + text_file.name, + json_file.name, + owl_file.name, + force_filetype=FileTypeAPI.Markdown + ) + + assert result == "test_job_123" + # Check that force_filetype was passed + call_args = mock_create.call_args + assert call_args[1]['force_filetype'] == FileTypeAPI.Markdown + + def test_create_job_invalid_text_file(self): + """Test job creation with invalid text file.""" + with pytest.raises(ClientException, match="Unstructured input text does not exist"): + self.cli.create_job( + "nonexistent.txt", + "test.json", + "test.owl" + ) + + def test_create_job_invalid_json_file(self): + """Test job creation with invalid JSON file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with pytest.raises(ClientException, match="Input ontology JSON file does not exist"): + self.cli.create_job( + text_file.name, + "nonexistent.json", + "test.owl" + ) + + def test_create_job_invalid_owl_file(self): + """Test job creation with invalid OWL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with pytest.raises(ClientException, match="Input ontology OWL file does not exist"): + self.cli.create_job( + text_file.name, + json_file.name, + "nonexistent.owl" + ) + + def test_create_job_invalid_wildcards_file(self): + """Test job creation with invalid wildcards file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file: + owl_file.write('test') + owl_file.flush() + + with pytest.raises(ClientException, match="Input wildcards file does not exist"): + self.cli.create_job( + text_file.name, + json_file.name, + owl_file.name, + "nonexistent.wildcards" + ) + + def test_create_job_none_wildcards(self): + """Test job creation with None wildcards.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as text_file: + text_file.write("Test text content") + text_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as json_file: + json_file.write('{"ontology": "data"}') + json_file.flush() + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.owl') as owl_file: + owl_file.write('test') + owl_file.flush() + + with patch.object(self.cli._client, 'create_unstructured_to_kg_job') as mock_create: + mock_create.return_value = "test_job_123" + + result = self.cli.create_job( + text_file.name, + json_file.name, + owl_file.name, + None + ) + + assert result == "test_job_123" + mock_create.assert_called_once() + + +class TestTextoToKGCLIJobPolling: + """Test cases for TextoToKGCLI job polling methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_success(self, mock_client_class): + """Test successful job polling.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Completed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_failed(self, mock_client_class): + """Test job polling with failed job.""" + mock_client = Mock() + mock_client.get_job_status.return_value = JobStatus.Failed + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + with pytest.raises(ClientException, match="finished without completing"): + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_processing(self, mock_client_class): + """Test job polling with processing job.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_ready_for_processing(self, mock_client_class): + """Test job polling with ready for processing job.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.get_job_status.return_value = JobStatus.ReadyForProcessing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + + # Should not raise any exception + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_invalid_status(self, mock_client_class): + """Test job polling with invalid status.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.get_job_status.return_value = JobStatus.Created + + with pytest.raises(ClientException, match="is not processing"): + self.cli.poll_job_to_completion("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_poll_job_to_completion_poll_failed(self, mock_client_class): + """Test job polling where poll returns failed.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Failed + + with pytest.raises(ClientException, match="finished without completing"): + self.cli.poll_job_to_completion("test_job_123") + + +class TestTextoToKGCLIJobDownload: + """Test cases for TextoToKGCLI job download methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_download_job_result_success(self, mock_client_class): + """Test successful job result download.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_job_details = { + "unstructured_sources": ["test.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + mock_client.retrieve_unstructured_to_kg_files.return_value = True + + # Should not raise any exception + self.cli.download_job_result("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_download_job_result_no_input_files(self, mock_client_class): + """Test job result download with no input files.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_job_details = { + "unstructured_sources": [] + } + mock_client.get_job_details.return_value = mock_job_details + + with pytest.raises(ClientException, match="has no input files"): + self.cli.download_job_result("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_download_job_result_multiple_input_files(self, mock_client_class): + """Test job result download with multiple input files.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_job_details = { + "unstructured_sources": ["test1.txt", "test2.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + + with pytest.raises(ClientException, match="has more than one input files"): + self.cli.download_job_result("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_download_job_result_detailed_output(self, mock_client_class, capsys): + """Test job result download with detailed output.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + self.cli._detailed = True + + mock_job_details = { + "unstructured_sources": ["test.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + mock_client.retrieve_unstructured_to_kg_files.return_value = True + + self.cli.download_job_result("test_job_123") + + captured = capsys.readouterr() + assert "Job result was downloaded to:" in captured.out + + +class TestTextoToKGCLIJobManagement: + """Test cases for TextoToKGCLI job management methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_list_server_jobs_success(self, mock_client_class): + """Test successful server jobs listing.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_jobs = [ + {"id": "job1", "status": "Completed"}, + {"id": "job2", "status": "Processing"} + ] + mock_client.get_jobs_list.return_value = mock_jobs + + result = self.cli.list_server_jobs() + assert result == mock_jobs + mock_client.get_jobs_list.assert_called_once_with(is_benchmark=False) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_delete_server_job_success(self, mock_client_class): + """Test successful server job deletion.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.delete_job.return_value = True + + # Should not raise any exception + self.cli.delete_server_job("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.CleverSwarmClient') + def test_delete_server_job_failure(self, mock_client_class): + """Test server job deletion failure.""" + mock_client = Mock() + mock_client_class.return_value = mock_client + self.cli._client = mock_client + + mock_client.delete_job.return_value = False + + # Should not raise any exception, but should handle failure gracefully + self.cli.delete_server_job("test_job_123") + + +class TestTextoToKGCLIJobPrinting: + """Test cases for TextoToKGCLI job printing methods.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + @patch.object(TextoToKGCLI, 'list_server_jobs') + def test_print_server_jobs_basic(self, mock_list_jobs, capsys): + """Test printing server jobs in basic mode.""" + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL" + } + ] + mock_list_jobs.return_value = mock_jobs + + self.cli.print_server_jobs() + + captured = capsys.readouterr() + assert "Job ID: job1" in captured.out + assert "Status: Completed" in captured.out + + @patch.object(TextoToKGCLI, 'list_server_jobs') + def test_print_server_jobs_detailed(self, mock_list_jobs, capsys): + """Test printing server jobs in detailed mode.""" + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"] + } + ] + mock_list_jobs.return_value = mock_jobs + + self.cli.print_server_jobs(is_detailed=True) + + captured = capsys.readouterr() + assert "Job ID: job1" in captured.out + assert "Unstructured text input files:" in captured.out + assert "Ontology JSON input files:" in captured.out + assert "Ontology OWL/XML input files:" in captured.out + + @patch.object(TextoToKGCLI, 'list_server_jobs') + def test_print_server_jobs_with_wildcards(self, mock_list_jobs, capsys): + """Test printing server jobs with wildcards.""" + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithWildcards", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"], + "wildcards_sources": ["test.wildcards"] + } + ] + mock_list_jobs.return_value = mock_jobs + + self.cli.print_server_jobs(is_detailed=True) + + captured = capsys.readouterr() + assert "Wildcards input files:" in captured.out + + +class TestTextoToKGCLIMainFunction: + """Test cases for TextoToKGCLI main function.""" + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_list_jobs(self, mock_cli_class): + """Test main function with list jobs action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'list-kg-jobs-and-exit']): + main() + + mock_cli.print_server_jobs.assert_called_once_with(False) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_create_and_exit_missing_args(self, mock_cli_class): + """Test main function with create and exit action missing required args.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'create-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_create_download_kg_delete(self, mock_cli_class): + """Test main function with create-download-kg-delete action.""" + mock_cli = Mock() + mock_cli.create_job.return_value = "test_job_123" + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'create-download-kg-delete', + '--unstructured_text', 'test.txt', + '--ontology_json', 'test.json', + '--ontology_owl', 'test.owl' + ]): + main() + + mock_cli.create_job.assert_called_once() + mock_cli.poll_job_to_completion.assert_called_once_with("test_job_123") + mock_cli.download_job_result.assert_called_once_with("test_job_123") + mock_cli.delete_server_job.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_client_exception(self, mock_cli_class): + """Test main function with client exception.""" + mock_cli = Mock() + mock_cli.create_job.side_effect = ClientException("Test error") + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'create-and-exit', + '--unstructured_text', 'test.txt', + '--ontology_json', 'test.json', + '--ontology_owl', 'test.owl' + ]): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_unexpected_exception(self, mock_cli_class): + """Test main function with unexpected exception.""" + mock_cli = Mock() + mock_cli.create_job.side_effect = Exception("Unexpected error") + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'create-and-exit', + '--unstructured_text', 'test.txt', + '--ontology_json', 'test.json', + '--ontology_owl', 'test.owl' + ]): + with pytest.raises(SystemExit): + main() + + +class TestTextoToKGCLIIntegration: + """Integration test cases for TextoToKGCLI.""" + + def test_full_workflow_simulation(self): + """Test a complete workflow simulation.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + # Test that client is properly initialized + assert cli._client is not None + assert cli._input_prefix == Path(temp_dir).resolve() + assert cli._output_prefix == Path(temp_dir).resolve() + + def test_path_resolution(self): + """Test path resolution functionality.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + # Test that paths are resolved correctly + assert cli._input_prefix.is_absolute() + assert cli._output_prefix.is_absolute() + + def test_file_validation_integration(self): + """Test file validation in integration context.""" + with tempfile.TemporaryDirectory() as temp_dir: + cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + # Create test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + + # Test that files are found correctly + assert text_file.exists() + assert json_file.exists() + assert owl_file.exists() + + # Test job creation with real files + with patch.object(cli._client, 'create_unstructured_to_kg_job') as mock_create: + mock_create.return_value = "test_job_123" + + result = cli.create_job( + str(text_file), + str(json_file), + str(owl_file) + ) + + assert result == "test_job_123" diff --git a/tests/test_cswarm_text_to_kg_client_extended.py b/tests/test_cswarm_text_to_kg_client_extended.py new file mode 100644 index 0000000..9c62ee2 --- /dev/null +++ b/tests/test_cswarm_text_to_kg_client_extended.py @@ -0,0 +1,454 @@ +""" +Extended unit tests for cswarm_text_to_kg_client module to achieve higher coverage. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch +import pytest + +from cleverswarm_python_client.cswarm_text_to_kg_client import TextoToKGCLI, main +from cleverswarm_python_client.libs.exceptions import ClientException +from cleverswarm_python_client.libs.job_enums import JobStatus + + +class TestTextoToKGCLIExtended: + """Extended test cases for TextoToKGCLI to achieve higher coverage.""" + + def setup_method(self): + """Set up test CLI.""" + with tempfile.TemporaryDirectory() as temp_dir: + self.cli = TextoToKGCLI( + "http://localhost:8000/api/v0/", + input_prefix=temp_dir, + output_prefix=temp_dir + ) + + def test_create_job_with_wildcards_and_force_filetype(self): + """Test job creation with wildcards and force filetype.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + wildcards_file = Path(temp_dir) / "test.wildcards" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + wildcards_file.write_text("wildcard1\nwildcard2") + + with patch.object(self.cli, '_client') as mock_client: + mock_client.create_unstructured_to_kg_wildcards_job.return_value = "test_job_123" + + result = self.cli.create_job( + str(text_file), + str(json_file), + str(owl_file), + wildcards=str(wildcards_file), + force_filetype="Markdown" + ) + + assert result == "test_job_123" + mock_client.create_unstructured_to_kg_wildcards_job.assert_called_once() + + def test_create_job_with_force_filetype_enum(self): + """Test job creation with force filetype as enum.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create test files + text_file = Path(temp_dir) / "test.txt" + json_file = Path(temp_dir) / "test.json" + owl_file = Path(temp_dir) / "test.owl" + + text_file.write_text("Test content") + json_file.write_text('{"test": "data"}') + owl_file.write_text('test') + + with patch.object(self.cli, '_client') as mock_client: + mock_client.create_unstructured_to_kg_job.return_value = "test_job_123" + + from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI + + result = self.cli.create_job( + str(text_file), + str(json_file), + str(owl_file), + force_filetype=FileTypeAPI.Markdown + ) + + assert result == "test_job_123" + mock_client.create_unstructured_to_kg_job.assert_called_once() + + def test_download_job_result_with_single_file(self): + """Test downloading job result with single file.""" + with patch.object(self.cli, '_client') as mock_client: + mock_job_details = { + "unstructured_sources": ["test.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + mock_client.retrieve_unstructured_to_kg_files.return_value = True + + result = self.cli.download_job_result("test_job_123") + + assert result is None + mock_client.get_job_details.assert_called_once_with("test_job_123") + mock_client.retrieve_unstructured_to_kg_files.assert_called_once() + + def test_download_job_result_with_multiple_files_raises_exception(self): + """Test downloading job result with multiple files raises exception.""" + with patch.object(self.cli, '_client') as mock_client: + mock_job_details = { + "unstructured_sources": ["test1.txt", "test2.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + + with pytest.raises(ClientException, match="has more than one input files"): + self.cli.download_job_result("test_job_123") + + def test_download_job_result_with_no_files_raises_exception(self): + """Test downloading job result with no files raises exception.""" + with patch.object(self.cli, '_client') as mock_client: + mock_job_details = { + "unstructured_sources": [] + } + mock_client.get_job_details.return_value = mock_job_details + + with pytest.raises(ClientException, match="has no input files"): + self.cli.download_job_result("test_job_123") + + def test_download_job_result_detailed_output(self): + """Test downloading job result with detailed output.""" + with patch.object(self.cli, '_client') as mock_client: + self.cli._detailed = True + mock_job_details = { + "unstructured_sources": ["test.txt"] + } + mock_client.get_job_details.return_value = mock_job_details + mock_client.retrieve_unstructured_to_kg_files.return_value = True + + with patch('builtins.print') as mock_print: + result = self.cli.download_job_result("test_job_123") + + assert result is None + mock_print.assert_called() + + def test_list_server_jobs_success(self): + """Test listing server jobs successfully.""" + with patch.object(self.cli, '_client') as mock_client: + mock_jobs = [ + {"id": "job1", "type": "UnstructuredWithOntology"}, + {"id": "job2", "type": "UnstructuredWithWildcards"} + ] + mock_client.get_jobs_list.return_value = mock_jobs + + result = self.cli.list_server_jobs() + + assert len(result) == 2 + assert result[0]["id"] == "job1" + mock_client.get_jobs_list.assert_called_once_with(is_benchmark=False) + + def test_print_server_jobs_with_empty_list(self): + """Test printing server jobs with empty list.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.get_jobs_list.return_value = [] + + with patch('builtins.print') as mock_print: + self.cli.print_server_jobs() + # Should not raise any exception + + def test_print_server_jobs_with_jobs(self): + """Test printing server jobs with actual jobs.""" + with patch.object(self.cli, '_client') as mock_client: + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"] + } + ] + mock_client.get_jobs_list.return_value = mock_jobs + + with patch('builtins.print') as mock_print: + self.cli.print_server_jobs() + assert mock_print.call_count > 0 + + def test_print_server_jobs_detailed_with_wildcards(self): + """Test printing server jobs with detailed output including wildcards.""" + with patch.object(self.cli, '_client') as mock_client: + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithWildcards", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"], + "wildcards_sources": ["test.wildcards"] + } + ] + mock_client.get_jobs_list.return_value = mock_jobs + + with patch('builtins.print') as mock_print: + self.cli.print_server_jobs(is_detailed=True) + assert mock_print.call_count > 0 + + def test_print_server_jobs_detailed_without_wildcards(self): + """Test printing server jobs with detailed output without wildcards.""" + with patch.object(self.cli, '_client') as mock_client: + mock_jobs = [ + { + "id": "job1", + "created": "2023-01-01T00:00:00Z", + "type": "UnstructuredWithOntology", + "status": "Completed", + "retries_count": 0, + "file_type": "JSONL", + "unstructured_sources": ["test.txt"], + "ontology_sources": ["test.json"], + "ontology_spec_sources": ["test.owl"] + } + ] + mock_client.get_jobs_list.return_value = mock_jobs + + with patch('builtins.print') as mock_print: + self.cli.print_server_jobs(is_detailed=True) + assert mock_print.call_count > 0 + + def test_delete_server_job_success(self): + """Test successful server job deletion.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.delete_job.return_value = True + + with patch('builtins.print') as mock_print: + self.cli.delete_server_job("test_job_123") + mock_print.assert_called_with("Job was successfully deleted") + + def test_delete_server_job_failure(self): + """Test failed server job deletion.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.delete_job.return_value = False + + with patch('builtins.print') as mock_print: + self.cli.delete_server_job("test_job_123") + mock_print.assert_called_with("Failed to delete Job") + + def test_poll_job_to_completion_with_ready_for_processing(self): + """Test polling job that is ready for processing.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.get_job_status.return_value = JobStatus.ReadyForProcessing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + + result = self.cli.poll_job_to_completion("test_job_123") + + assert result is None # Should not raise exception + mock_client.get_job_status.assert_called_once_with("test_job_123") + mock_client.poll_job_ready_or_failed.assert_called_once_with("test_job_123") + + def test_poll_job_to_completion_with_processing(self): + """Test polling job that is processing.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.return_value = JobStatus.Completed + + result = self.cli.poll_job_to_completion("test_job_123") + + assert result is None # Should not raise exception + mock_client.get_job_status.assert_called_once_with("test_job_123") + mock_client.poll_job_ready_or_failed.assert_called_once_with("test_job_123") + + def test_poll_job_to_completion_with_invalid_status(self): + """Test polling job with invalid status.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.get_job_status.return_value = "InvalidStatus" + + with pytest.raises(ClientException, match="is not processing, nor is it ready for processing"): + self.cli.poll_job_to_completion("test_job_123") + + def test_poll_job_to_completion_with_poll_failure(self): + """Test polling job with poll failure.""" + with patch.object(self.cli, '_client') as mock_client: + mock_client.get_job_status.return_value = JobStatus.Processing + mock_client.poll_job_ready_or_failed.side_effect = ClientException("Poll failed") + + with pytest.raises(ClientException, match="Poll failed"): + self.cli.poll_job_to_completion("test_job_123") + + +class TestTextoToKGCLIMainFunctionExtended: + """Extended test cases for TextoToKGCLI main function to achieve higher coverage.""" + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_delete_and_exit(self, mock_cli_class): + """Test main function with delete and exit action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'delete-and-exit', '--job_id', 'test_job_123']): + main() + + mock_cli.delete_server_job.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_delete_and_exit_missing_job_id(self, mock_cli_class): + """Test main function with delete and exit action missing job_id.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'delete-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_download_kg_and_exit(self, mock_cli_class): + """Test main function with download-kg-and-exit action.""" + mock_cli = Mock() + mock_cli.download_job_result.return_value = True + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'download-kg-and-exit', '--job_id', 'test_job_123']): + main() + + mock_cli.download_job_result.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_download_kg_and_exit_missing_job_id(self, mock_cli_class): + """Test main function with download-kg-and-exit action missing job_id.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'download-kg-and-exit']): + with pytest.raises(SystemExit): + main() + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_list_kg_jobs_and_exit(self, mock_cli_class): + """Test main function with list-kg-jobs-and-exit action.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'list-kg-jobs-and-exit']): + main() + + mock_cli.print_server_jobs.assert_called_once_with(False) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_create_download_kg(self, mock_cli_class): + """Test main function with create-download-kg action.""" + mock_cli = Mock() + mock_cli.create_job.return_value = "test_job_123" + mock_cli.poll_job_to_completion.return_value = None + mock_cli.download_job_result.return_value = True + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'create-download-kg', + '--unstructured_text', 'test.txt', + '--ontology_json', 'test.json', + '--ontology_owl', 'test.owl' + ]): + main() + + mock_cli.create_job.assert_called_once() + mock_cli.poll_job_to_completion.assert_called_once_with("test_job_123") + mock_cli.download_job_result.assert_called_once_with("test_job_123") + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_with_wildcards(self, mock_cli_class): + """Test main function with wildcards.""" + mock_cli = Mock() + mock_cli.create_job.return_value = "test_job_123" + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'create-and-exit', + '--unstructured_text', 'test.txt', + '--ontology_json', 'test.json', + '--ontology_owl', 'test.owl', + '--wildcards', 'test.wildcards' + ]): + main() + + mock_cli.create_job.assert_called_once() + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_with_detailed_flag(self, mock_cli_class): + """Test main function with detailed flag.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'list-kg-jobs-and-exit', '--detailed']): + main() + + mock_cli.print_server_jobs.assert_called_once_with(True) + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_with_custom_server_url(self, mock_cli_class): + """Test main function with custom server URL.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'list-kg-jobs-and-exit', '--server_url', 'http://custom:8000/api/']): + main() + + mock_cli_class.assert_called_once() + call_args = mock_cli_class.call_args + assert call_args[1]['base_url'] == 'http://custom:8000/api/' + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_with_username_and_token(self, mock_cli_class): + """Test main function with username and token.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', ['cswarm_text_to_kg_client.py', '--action', 'list-kg-jobs-and-exit', '--username', 'testuser', '--token', 'testtoken']): + main() + + mock_cli_class.assert_called_once() + call_args = mock_cli_class.call_args + assert call_args[1]['username'] == 'testuser' + assert call_args[1]['token'] == 'testtoken' + + @patch('cleverswarm_python_client.cswarm_text_to_kg_client.TextoToKGCLI') + def test_main_with_custom_input_output_prefixes(self, mock_cli_class): + """Test main function with custom input and output prefixes.""" + mock_cli = Mock() + mock_cli_class.return_value = mock_cli + + with patch('sys.argv', [ + 'cswarm_text_to_kg_client.py', + '--action', 'list-kg-jobs-and-exit', + '--input_prefix', '/custom/input', + '--output_prefix', '/custom/output' + ]): + main() + + mock_cli_class.assert_called_once() + call_args = mock_cli_class.call_args + assert call_args[1]['input_prefix'] == '/custom/input' + assert call_args[1]['output_prefix'] == '/custom/output' diff --git a/tests/test_detailed_eval_extended.py b/tests/test_detailed_eval_extended.py new file mode 100644 index 0000000..5156b3b --- /dev/null +++ b/tests/test_detailed_eval_extended.py @@ -0,0 +1,239 @@ +""" +Extended unit tests for detailed_eval module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import tempfile +import pytest +from pathlib import Path +from unittest.mock import patch, Mock + +from cleverswarm_python_client.libs.evaluation.detailed_eval import ( + CustomEncoder, + read_jsonl, + write_jsonl, + load_ontology, + calculate_metrics, + normalize_string, + normalize_triple, + calculate_precision_recall_f1, + evaluate_entity_presence, +) + + +class TestDetailedEvalExtendedSimple: + """Extended test cases for detailed_eval module focusing on working tests.""" + + def test_custom_encoder_with_set(self): + """Test CustomEncoder with set objects.""" + encoder = CustomEncoder() + test_set = {1, 2, 3, "test"} + result = encoder.default(test_set) + assert result == [1, 2, 3, "test"] + + def test_custom_encoder_with_non_set(self): + """Test CustomEncoder with non-set objects.""" + encoder = CustomEncoder() + test_dict = {"key": "value"} + # Should call super().default() which will raise TypeError for non-serializable objects + with pytest.raises(TypeError): + encoder.default(test_dict) + + def test_read_jsonl_with_empty_file(self): + """Test reading empty JSONL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.write('') + f.flush() + + result = read_jsonl(f.name) + assert result == [] + + def test_read_jsonl_with_single_line(self): + """Test reading JSONL file with single line.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.write('{"key": "value"}\n') + f.flush() + + result = read_jsonl(f.name) + assert result == [{"key": "value"}] + + def test_read_jsonl_with_multiple_lines(self): + """Test reading JSONL file with multiple lines.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.write('{"key1": "value1"}\n{"key2": "value2"}\n') + f.flush() + + result = read_jsonl(f.name) + assert result == [{"key1": "value1"}, {"key2": "value2"}] + + def test_read_jsonl_with_invalid_json(self): + """Test reading JSONL file with invalid JSON.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.write('{"key": "value"\n') # Missing closing brace + f.flush() + + with pytest.raises(Exception): # Should raise JSON decode error + read_jsonl(f.name) + + def test_write_jsonl_with_empty_list(self): + """Test writing empty list to JSONL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.flush() + temp_file = f.name + + write_jsonl([], temp_file) + + with open(temp_file, 'r') as f: + content = f.read() + assert content == '' + + def test_write_jsonl_with_single_dict(self): + """Test writing single dict to JSONL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.flush() + temp_file = f.name + + data = [{"key": "value"}] + write_jsonl(data, temp_file) + + with open(temp_file, 'r') as f: + content = f.read() + assert content == '{"key": "value"}\n' + + def test_write_jsonl_with_multiple_dicts(self): + """Test writing multiple dicts to JSONL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.flush() + temp_file = f.name + + data = [{"key1": "value1"}, {"key2": "value2"}] + write_jsonl(data, temp_file) + + with open(temp_file, 'r') as f: + content = f.read() + assert content == '{"key1": "value1"}\n{"key2": "value2"}\n' + + def test_write_jsonl_with_sets(self): + """Test writing data with sets to JSONL file.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.jsonl') as f: + f.flush() + temp_file = f.name + + data = [{"set_data": {1, 2, 3}}] + write_jsonl(data, temp_file) + + with open(temp_file, 'r') as f: + content = f.read() + assert content == '{"set_data": [1, 2, 3]}\n' + + def test_normalize_string_with_parentheses(self): + """Test string normalization with parentheses.""" + test_string = "Test (with) parentheses" + result = normalize_string(test_string) + assert result == "testparenthese" # Actual behavior removes content in parentheses + + def test_normalize_string_with_comma(self): + """Test string normalization with comma.""" + test_string = "Test, with, comma" + result = normalize_string(test_string) + assert result == "test" # Actual behavior removes non-alphanumeric characters + + def test_normalize_string_with_the_prefix(self): + """Test string normalization with 'the' prefix.""" + test_string = "The test string" + result = normalize_string(test_string) + assert result == "teststring" + + def test_normalize_string_with_trailing_s(self): + """Test string normalization with trailing 's'.""" + test_string = "tests" + result = normalize_string(test_string) + assert result == "test" + + def test_normalize_triple_with_empty_strings(self): + """Test triple normalization with empty strings.""" + result = normalize_triple("", "", "") + assert result == "" + + def test_normalize_triple_with_non_string_object(self): + """Test triple normalization with non-string object.""" + # The function expects strings, so this will raise an error + with pytest.raises(TypeError): + normalize_triple(123, "test", "value") + + def test_calculate_metrics_with_zero_denominator(self): + """Test metrics calculation with zero denominator.""" + result = calculate_metrics(0, 0, 0) + assert result == (0.0, 0.0, 0.0) + + def test_calculate_metrics_with_perfect_match(self): + """Test metrics calculation with perfect match.""" + result = calculate_metrics(10, 0, 0) + assert result == (1.0, 1.0, 1.0) + + def test_calculate_metrics_with_partial_match(self): + """Test metrics calculation with partial match.""" + result = calculate_metrics(5, 2, 3) + precision, recall, f1 = result + assert precision == round(5/7, 2) # Function rounds to 2 decimal places + assert recall == round(5/8, 2) + assert f1 == round(2 * (5/7) * (5/8) / ((5/7) + (5/8)), 2) + + def test_evaluate_entity_presence_with_entities(self): + """Test entity presence evaluation with entities.""" + ground_truth = { + "triples": [ + {"sub": "entity1", "rel": "relation1", "obj": "entity2"} + ] + } + response = { + "ents": [ + {"text": "entity1", "label": "Entity1"}, + {"text": "entity2", "label": "Entity2"} + ] + } + result = evaluate_entity_presence(ground_truth, response) + assert "precision" in result + assert "recall" in result + assert "f1" in result + + def test_calculate_precision_recall_f1_with_empty_sets(self): + """Test precision/recall/F1 calculation with empty sets.""" + gold = set() + pred = set() + precision, recall, f1 = calculate_precision_recall_f1(gold, pred) + assert precision == 0.0 + assert recall == 0.0 + assert f1 == 0.0 + + def test_calculate_precision_recall_f1_with_perfect_match(self): + """Test precision/recall/F1 calculation with perfect match.""" + gold = {1, 2, 3} + pred = {1, 2, 3} + precision, recall, f1 = calculate_precision_recall_f1(gold, pred) + assert precision == 1.0 + assert recall == 1.0 + assert f1 == 1.0 + + def test_calculate_precision_recall_f1_with_partial_match(self): + """Test precision/recall/F1 calculation with partial match.""" + gold = {1, 2, 3, 4} + pred = {1, 2, 3, 5} + precision, recall, f1 = calculate_precision_recall_f1(gold, pred) + assert precision == 0.75 # 3/4 + assert recall == 0.75 # 3/4 + assert f1 == 0.75 # 2 * 0.75 * 0.75 / (0.75 + 0.75) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py new file mode 100644 index 0000000..1c8a413 --- /dev/null +++ b/tests/test_exceptions.py @@ -0,0 +1,201 @@ +""" +Unit tests for exceptions module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest + +from cleverswarm_python_client.libs.exceptions import ( + BaseUrlMissingException, + ClientException, + ConnectionErrorException, + ConnectionTimeoutException, + InvalidFileException, + InvalidUsernameOrPasswordException, + JobNotFoundException, + RequestErrorException, + UnauthorizedException, + UnexpectedConditionException, +) + + +class TestClientException: + """Test cases for ClientException base class.""" + + def test_client_exception_inheritance(self): + """Test that ClientException inherits from Exception.""" + assert issubclass(ClientException, Exception) + + def test_client_exception_instantiation(self): + """Test that ClientException can be instantiated with a message.""" + message = "Test client exception" + exc = ClientException(message) + assert str(exc) == message + + def test_client_exception_no_message(self): + """Test that ClientException can be instantiated without a message.""" + exc = ClientException() + assert str(exc) == "" + + +class TestBaseUrlMissingException: + """Test cases for BaseUrlMissingException.""" + + def test_inheritance(self): + """Test that BaseUrlMissingException inherits from ClientException.""" + assert issubclass(BaseUrlMissingException, ClientException) + + def test_instantiation(self): + """Test that BaseUrlMissingException can be instantiated.""" + exc = BaseUrlMissingException("Base URL is missing") + assert str(exc) == "Base URL is missing" + + +class TestInvalidUsernameOrPasswordException: + """Test cases for InvalidUsernameOrPasswordException.""" + + def test_inheritance(self): + """Test that InvalidUsernameOrPasswordException inherits from ClientException.""" + assert issubclass(InvalidUsernameOrPasswordException, ClientException) + + def test_instantiation(self): + """Test that InvalidUsernameOrPasswordException can be instantiated.""" + exc = InvalidUsernameOrPasswordException("Invalid credentials") + assert str(exc) == "Invalid credentials" + + +class TestUnexpectedConditionException: + """Test cases for UnexpectedConditionException.""" + + def test_inheritance(self): + """Test that UnexpectedConditionException inherits from ClientException.""" + assert issubclass(UnexpectedConditionException, ClientException) + + def test_instantiation(self): + """Test that UnexpectedConditionException can be instantiated.""" + exc = UnexpectedConditionException("Unexpected condition occurred") + assert str(exc) == "Unexpected condition occurred" + + +class TestRequestErrorException: + """Test cases for RequestErrorException.""" + + def test_inheritance(self): + """Test that RequestErrorException inherits from ClientException.""" + assert issubclass(RequestErrorException, ClientException) + + def test_instantiation(self): + """Test that RequestErrorException can be instantiated.""" + exc = RequestErrorException("Request error occurred") + assert str(exc) == "Request error occurred" + + +class TestConnectionErrorException: + """Test cases for ConnectionErrorException.""" + + def test_inheritance(self): + """Test that ConnectionErrorException inherits from ClientException.""" + assert issubclass(ConnectionErrorException, ClientException) + + def test_instantiation(self): + """Test that ConnectionErrorException can be instantiated.""" + exc = ConnectionErrorException("Connection failed") + assert str(exc) == "Connection failed" + + +class TestConnectionTimeoutException: + """Test cases for ConnectionTimeoutException.""" + + def test_inheritance(self): + """Test that ConnectionTimeoutException inherits from ClientException.""" + assert issubclass(ConnectionTimeoutException, ClientException) + + def test_instantiation(self): + """Test that ConnectionTimeoutException can be instantiated.""" + exc = ConnectionTimeoutException("Connection timed out") + assert str(exc) == "Connection timed out" + + +class TestJobNotFoundException: + """Test cases for JobNotFoundException.""" + + def test_inheritance(self): + """Test that JobNotFoundException inherits from ClientException.""" + assert issubclass(JobNotFoundException, ClientException) + + def test_instantiation(self): + """Test that JobNotFoundException can be instantiated.""" + exc = JobNotFoundException("Job not found") + assert str(exc) == "Job not found" + + +class TestUnauthorizedException: + """Test cases for UnauthorizedException.""" + + def test_inheritance(self): + """Test that UnauthorizedException inherits from ClientException.""" + assert issubclass(UnauthorizedException, ClientException) + + def test_instantiation(self): + """Test that UnauthorizedException can be instantiated.""" + exc = UnauthorizedException("Unauthorized access") + assert str(exc) == "Unauthorized access" + + +class TestInvalidFileException: + """Test cases for InvalidFileException.""" + + def test_inheritance(self): + """Test that InvalidFileException inherits from ClientException.""" + assert issubclass(InvalidFileException, ClientException) + + def test_instantiation(self): + """Test that InvalidFileException can be instantiated.""" + exc = InvalidFileException("Invalid file") + assert str(exc) == "Invalid file" + + +class TestExceptionHierarchy: + """Test cases for exception hierarchy and behavior.""" + + def test_exception_catching(self): + """Test that specific exceptions can be caught by their parent.""" + with pytest.raises(ClientException): + raise BaseUrlMissingException("Test") + + with pytest.raises(ClientException): + raise InvalidUsernameOrPasswordException("Test") + + with pytest.raises(ClientException): + raise UnexpectedConditionException("Test") + + def test_exception_chain(self): + """Test that exceptions can be chained properly.""" + try: + try: + raise ConnectionErrorException("Connection failed") + except ConnectionErrorException as e: + raise ClientException("Higher level error") from e + except ClientException as e: + assert isinstance(e.__cause__, ConnectionErrorException) + assert str(e.__cause__) == "Connection failed" + + def test_exception_with_custom_attributes(self): + """Test that exceptions can have custom attributes.""" + exc = ClientException("Test message") + exc.custom_attr = "custom_value" + assert exc.custom_attr == "custom_value" diff --git a/tests/test_file_type_enum.py b/tests/test_file_type_enum.py new file mode 100644 index 0000000..9338027 --- /dev/null +++ b/tests/test_file_type_enum.py @@ -0,0 +1,208 @@ +""" +Unit tests for file_type_enum module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest + +from cleverswarm_python_client.libs.file_type_enum import FileType, FileTypeAPI + + +class TestFileTypeAPI: + """Test cases for FileTypeAPI enum.""" + + def test_enum_values(self): + """Test that FileTypeAPI has the correct values.""" + assert FileTypeAPI.AutoDetect == "AutoDetect" + assert FileTypeAPI.Markdown == "Markdown" + assert FileTypeAPI.PlainText == "PlainText" + + def test_enum_membership(self): + """Test that FileTypeAPI members can be checked for membership.""" + # Test enum member membership (works in all Python versions) + assert FileTypeAPI.AutoDetect in FileTypeAPI + assert FileTypeAPI.Markdown in FileTypeAPI + assert FileTypeAPI.PlainText in FileTypeAPI + + # Test string value membership (Python 3.12+ only) + import sys + if sys.version_info >= (3, 12): + assert "AutoDetect" in FileTypeAPI + assert "Markdown" in FileTypeAPI + assert "PlainText" in FileTypeAPI + assert "InvalidType" not in FileTypeAPI + + def test_enum_iteration(self): + """Test that FileTypeAPI can be iterated over.""" + values = list(FileTypeAPI) + expected_values = ["AutoDetect", "Markdown", "PlainText"] + assert len(values) == 3 + for value in expected_values: + assert value in values + + def test_enum_comparison(self): + """Test that FileTypeAPI values can be compared.""" + assert FileTypeAPI.AutoDetect == "AutoDetect" + assert FileTypeAPI.Markdown == "Markdown" + assert FileTypeAPI.PlainText == "PlainText" + assert FileTypeAPI.AutoDetect != FileTypeAPI.Markdown + + def test_enum_string_representation(self): + """Test string representation of FileTypeAPI values.""" + assert str(FileTypeAPI.AutoDetect) == "AutoDetect" + assert str(FileTypeAPI.Markdown) == "Markdown" + assert str(FileTypeAPI.PlainText) == "PlainText" + + def test_enum_repr(self): + """Test repr representation of FileTypeAPI values.""" + # Python 3.12+ uses a different repr format for enums + assert "FileTypeAPI.AutoDetect" in repr(FileTypeAPI.AutoDetect) + assert "FileTypeAPI.Markdown" in repr(FileTypeAPI.Markdown) + assert "FileTypeAPI.PlainText" in repr(FileTypeAPI.PlainText) + + def test_enum_inheritance(self): + """Test that FileTypeAPI inherits from StrEnum.""" + from enum import StrEnum + assert issubclass(FileTypeAPI, StrEnum) + + def test_enum_equality_with_string(self): + """Test that FileTypeAPI values are equal to their string representations.""" + assert FileTypeAPI.AutoDetect == "AutoDetect" + assert FileTypeAPI.Markdown == "Markdown" + assert FileTypeAPI.PlainText == "PlainText" + + def test_enum_case_sensitivity(self): + """Test that FileTypeAPI values are case sensitive.""" + assert FileTypeAPI.AutoDetect != "autodetect" + assert FileTypeAPI.Markdown != "markdown" + assert FileTypeAPI.PlainText != "plaintext" + + +class TestFileType: + """Test cases for FileType enum.""" + + def test_enum_values(self): + """Test that FileType has the correct values.""" + assert FileType.JSONL == "JSONL" + assert FileType.Markdown == "Markdown" + assert FileType.PlainText == "PlainText" + assert FileType.Unknown == "Unknown" + + def test_enum_membership(self): + """Test that FileType members can be checked for membership.""" + # Test enum member membership (works in all Python versions) + assert FileType.JSONL in FileType + assert FileType.Markdown in FileType + assert FileType.PlainText in FileType + assert FileType.Unknown in FileType + + # Test string value membership (Python 3.12+ only) + import sys + if sys.version_info >= (3, 12): + assert "JSONL" in FileType + assert "Markdown" in FileType + assert "PlainText" in FileType + assert "Unknown" in FileType + assert "InvalidType" not in FileType + + def test_enum_iteration(self): + """Test that FileType can be iterated over.""" + values = list(FileType) + expected_values = ["JSONL", "Markdown", "PlainText", "Unknown"] + assert len(values) == 4 + for value in expected_values: + assert value in values + + def test_enum_comparison(self): + """Test that FileType values can be compared.""" + assert FileType.JSONL == "JSONL" + assert FileType.Markdown == "Markdown" + assert FileType.PlainText == "PlainText" + assert FileType.Unknown == "Unknown" + assert FileType.JSONL != FileType.Markdown + + def test_enum_string_representation(self): + """Test string representation of FileType values.""" + assert str(FileType.JSONL) == "JSONL" + assert str(FileType.Markdown) == "Markdown" + assert str(FileType.PlainText) == "PlainText" + assert str(FileType.Unknown) == "Unknown" + + def test_enum_repr(self): + """Test repr representation of FileType values.""" + # Python 3.12+ uses a different repr format for enums + assert "FileType.JSONL" in repr(FileType.JSONL) + assert "FileType.Markdown" in repr(FileType.Markdown) + assert "FileType.PlainText" in repr(FileType.PlainText) + assert "FileType.Unknown" in repr(FileType.Unknown) + + def test_enum_inheritance(self): + """Test that FileType inherits from StrEnum.""" + from enum import StrEnum + assert issubclass(FileType, StrEnum) + + def test_enum_equality_with_string(self): + """Test that FileType values are equal to their string representations.""" + assert FileType.JSONL == "JSONL" + assert FileType.Markdown == "Markdown" + assert FileType.PlainText == "PlainText" + assert FileType.Unknown == "Unknown" + + def test_enum_case_sensitivity(self): + """Test that FileType values are case sensitive.""" + assert FileType.JSONL != "jsonl" + assert FileType.Markdown != "markdown" + assert FileType.PlainText != "plaintext" + assert FileType.Unknown != "unknown" + + +class TestEnumInteractions: + """Test cases for interactions between FileType and FileTypeAPI enums.""" + + def test_different_enum_types(self): + """Test that FileType and FileTypeAPI are different enum types.""" + assert FileType.Markdown == FileTypeAPI.Markdown # Same string value + assert FileType.PlainText == FileTypeAPI.PlainText # Same string value + assert FileType.Markdown is not FileTypeAPI.Markdown # Different objects + + def test_enum_type_checking(self): + """Test that enum types can be checked.""" + assert isinstance(FileType.Markdown, FileType) + assert isinstance(FileTypeAPI.Markdown, FileTypeAPI) + assert not isinstance(FileType.Markdown, FileTypeAPI) + assert not isinstance(FileTypeAPI.Markdown, FileType) + + def test_enum_value_consistency(self): + """Test that shared values between enums are consistent.""" + assert FileType.Markdown == "Markdown" + assert FileTypeAPI.Markdown == "Markdown" + assert FileType.PlainText == "PlainText" + assert FileTypeAPI.PlainText == "PlainText" + + def test_enum_unique_values(self): + """Test that each enum has unique values.""" + file_type_values = [member.value for member in FileType] + file_type_api_values = [member.value for member in FileTypeAPI] + + # Check that values within each enum are unique + assert len(file_type_values) == len(set(file_type_values)) + assert len(file_type_api_values) == len(set(file_type_api_values)) + + # Check that there are some overlapping values + common_values = set(file_type_values) & set(file_type_api_values) + assert "Markdown" in common_values + assert "PlainText" in common_values diff --git a/tests/test_job_enums.py b/tests/test_job_enums.py new file mode 100644 index 0000000..20f6796 --- /dev/null +++ b/tests/test_job_enums.py @@ -0,0 +1,291 @@ +""" +Unit tests for job_enums module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest + +from cleverswarm_python_client.libs.job_enums import JobStatus, JobType + + +class TestJobType: + """Test cases for JobType enum.""" + + def test_enum_values(self): + """Test that JobType has the correct values.""" + assert JobType.UnstructuredWithOntology == "UnstructuredWithOntology" + assert JobType.UnstructuredWithWildcards == "UnstructuredWithWildcards" + assert JobType.Benchmark == "Benchmark" + + def test_enum_membership(self): + """Test that JobType members can be checked for membership.""" + # Test enum member membership (works in all Python versions) + assert JobType.UnstructuredWithOntology in JobType + assert JobType.UnstructuredWithWildcards in JobType + assert JobType.Benchmark in JobType + + # Test string value membership (Python 3.12+ only) + import sys + if sys.version_info >= (3, 12): + assert "UnstructuredWithOntology" in JobType + assert "UnstructuredWithWildcards" in JobType + assert "Benchmark" in JobType + assert "InvalidType" not in JobType + + def test_enum_iteration(self): + """Test that JobType can be iterated over.""" + values = list(JobType) + expected_values = ["UnstructuredWithOntology", "UnstructuredWithWildcards", "Benchmark"] + assert len(values) == 3 + for value in expected_values: + assert value in values + + def test_enum_comparison(self): + """Test that JobType values can be compared.""" + assert JobType.UnstructuredWithOntology == "UnstructuredWithOntology" + assert JobType.UnstructuredWithWildcards == "UnstructuredWithWildcards" + assert JobType.Benchmark == "Benchmark" + assert JobType.UnstructuredWithOntology != JobType.Benchmark + + def test_enum_string_representation(self): + """Test string representation of JobType values.""" + assert str(JobType.UnstructuredWithOntology) == "UnstructuredWithOntology" + assert str(JobType.UnstructuredWithWildcards) == "UnstructuredWithWildcards" + assert str(JobType.Benchmark) == "Benchmark" + + def test_enum_repr(self): + """Test repr representation of JobType values.""" + # Python 3.12+ uses a different repr format for enums + assert "JobType.UnstructuredWithOntology" in repr(JobType.UnstructuredWithOntology) + assert "JobType.UnstructuredWithWildcards" in repr(JobType.UnstructuredWithWildcards) + assert "JobType.Benchmark" in repr(JobType.Benchmark) + + def test_enum_inheritance(self): + """Test that JobType inherits from StrEnum.""" + from enum import StrEnum + assert issubclass(JobType, StrEnum) + + def test_enum_equality_with_string(self): + """Test that JobType values are equal to their string representations.""" + assert JobType.UnstructuredWithOntology == "UnstructuredWithOntology" + assert JobType.UnstructuredWithWildcards == "UnstructuredWithWildcards" + assert JobType.Benchmark == "Benchmark" + + def test_enum_case_sensitivity(self): + """Test that JobType values are case sensitive.""" + assert JobType.UnstructuredWithOntology != "unstructuredwithontology" + assert JobType.UnstructuredWithWildcards != "unstructuredwithwildcards" + assert JobType.Benchmark != "benchmark" + + def test_enum_ordering(self): + """Test that JobType values can be ordered.""" + job_types = [JobType.Benchmark, JobType.UnstructuredWithOntology, JobType.UnstructuredWithWildcards] + sorted_types = sorted(job_types) + expected_order = ["Benchmark", "UnstructuredWithOntology", "UnstructuredWithWildcards"] + assert [str(t) for t in sorted_types] == expected_order + + +class TestJobStatus: + """Test cases for JobStatus enum.""" + + def test_enum_values(self): + """Test that JobStatus has the correct values.""" + assert JobStatus.Created == "Created" + assert JobStatus.ReadyForProcessing == "ReadyForProcessing" + assert JobStatus.Processing == "Processing" + assert JobStatus.Completed == "Completed" + assert JobStatus.Failed == "Failed" + + def test_enum_membership(self): + """Test that JobStatus members can be checked for membership.""" + # Test enum member membership (works in all Python versions) + assert JobStatus.Created in JobStatus + assert JobStatus.ReadyForProcessing in JobStatus + assert JobStatus.Processing in JobStatus + assert JobStatus.Completed in JobStatus + assert JobStatus.Failed in JobStatus + + # Test string value membership (Python 3.12+ only) + import sys + if sys.version_info >= (3, 12): + assert "Created" in JobStatus + assert "ReadyForProcessing" in JobStatus + assert "Processing" in JobStatus + assert "Completed" in JobStatus + assert "Failed" in JobStatus + assert "InvalidStatus" not in JobStatus + + def test_enum_iteration(self): + """Test that JobStatus can be iterated over.""" + values = list(JobStatus) + expected_values = ["Created", "ReadyForProcessing", "Processing", "Completed", "Failed"] + assert len(values) == 5 + for value in expected_values: + assert value in values + + def test_enum_comparison(self): + """Test that JobStatus values can be compared.""" + assert JobStatus.Created == "Created" + assert JobStatus.ReadyForProcessing == "ReadyForProcessing" + assert JobStatus.Processing == "Processing" + assert JobStatus.Completed == "Completed" + assert JobStatus.Failed == "Failed" + assert JobStatus.Created != JobStatus.Completed + + def test_enum_string_representation(self): + """Test string representation of JobStatus values.""" + assert str(JobStatus.Created) == "Created" + assert str(JobStatus.ReadyForProcessing) == "ReadyForProcessing" + assert str(JobStatus.Processing) == "Processing" + assert str(JobStatus.Completed) == "Completed" + assert str(JobStatus.Failed) == "Failed" + + def test_enum_repr(self): + """Test repr representation of JobStatus values.""" + # Python 3.12+ uses a different repr format for enums + assert "JobStatus.Created" in repr(JobStatus.Created) + assert "JobStatus.ReadyForProcessing" in repr(JobStatus.ReadyForProcessing) + assert "JobStatus.Processing" in repr(JobStatus.Processing) + assert "JobStatus.Completed" in repr(JobStatus.Completed) + assert "JobStatus.Failed" in repr(JobStatus.Failed) + + def test_enum_inheritance(self): + """Test that JobStatus inherits from StrEnum.""" + from enum import StrEnum + assert issubclass(JobStatus, StrEnum) + + def test_enum_equality_with_string(self): + """Test that JobStatus values are equal to their string representations.""" + assert JobStatus.Created == "Created" + assert JobStatus.ReadyForProcessing == "ReadyForProcessing" + assert JobStatus.Processing == "Processing" + assert JobStatus.Completed == "Completed" + assert JobStatus.Failed == "Failed" + + def test_enum_case_sensitivity(self): + """Test that JobStatus values are case sensitive.""" + assert JobStatus.Created != "created" + assert JobStatus.ReadyForProcessing != "readyforprocessing" + assert JobStatus.Processing != "processing" + assert JobStatus.Completed != "completed" + assert JobStatus.Failed != "failed" + + def test_enum_ordering(self): + """Test that JobStatus values can be ordered.""" + statuses = [JobStatus.Failed, JobStatus.Created, JobStatus.Completed, JobStatus.Processing, JobStatus.ReadyForProcessing] + sorted_statuses = sorted(statuses) + expected_order = ["Completed", "Created", "Failed", "Processing", "ReadyForProcessing"] + assert [str(s) for s in sorted_statuses] == expected_order + + def test_enum_workflow_states(self): + """Test that JobStatus represents a valid workflow.""" + # Test that we can represent a typical job workflow + workflow = [ + JobStatus.Created, + JobStatus.ReadyForProcessing, + JobStatus.Processing, + JobStatus.Completed + ] + + # All states should be valid + for status in workflow: + assert status in JobStatus + + # Test failure state + assert JobStatus.Failed in JobStatus + + +class TestEnumInteractions: + """Test cases for interactions between JobType and JobStatus enums.""" + + def test_different_enum_types(self): + """Test that JobType and JobStatus are different enum types.""" + # They should not be the same type + assert JobType.Benchmark is not JobStatus.Completed + + # But they can have the same string value (though they don't in this case) + assert JobType.Benchmark != JobStatus.Completed + + def test_enum_type_checking(self): + """Test that enum types can be checked.""" + assert isinstance(JobType.Benchmark, JobType) + assert isinstance(JobStatus.Completed, JobStatus) + assert not isinstance(JobType.Benchmark, JobStatus) + assert not isinstance(JobStatus.Completed, JobType) + + def test_enum_unique_values(self): + """Test that each enum has unique values.""" + job_type_values = [member.value for member in JobType] + job_status_values = [member.value for member in JobStatus] + + # Check that values within each enum are unique + assert len(job_type_values) == len(set(job_type_values)) + assert len(job_status_values) == len(set(job_status_values)) + + # Check that there are no overlapping values between the enums + common_values = set(job_type_values) & set(job_status_values) + assert len(common_values) == 0 + + def test_enum_usage_in_conditionals(self): + """Test that enums can be used in conditional statements.""" + job_type = JobType.Benchmark + job_status = JobStatus.Completed + + # Test equality conditions + if job_type == JobType.Benchmark: + assert True + else: + assert False + + if job_status == JobStatus.Completed: + assert True + else: + assert False + + def test_enum_usage_in_switch_like_statements(self): + """Test that enums can be used in switch-like statements.""" + def process_job_type(job_type): + if job_type == JobType.UnstructuredWithOntology: + return "processing_with_ontology" + elif job_type == JobType.UnstructuredWithWildcards: + return "processing_with_wildcards" + elif job_type == JobType.Benchmark: + return "processing_benchmark" + else: + return "unknown_type" + + assert process_job_type(JobType.UnstructuredWithOntology) == "processing_with_ontology" + assert process_job_type(JobType.UnstructuredWithWildcards) == "processing_with_wildcards" + assert process_job_type(JobType.Benchmark) == "processing_benchmark" + + def test_enum_usage_in_status_checks(self): + """Test that enums can be used in status checking functions.""" + def is_job_finished(status): + return status in [JobStatus.Completed, JobStatus.Failed] + + def is_job_running(status): + return status in [JobStatus.Processing, JobStatus.ReadyForProcessing] + + assert is_job_finished(JobStatus.Completed) + assert is_job_finished(JobStatus.Failed) + assert not is_job_finished(JobStatus.Processing) + assert not is_job_finished(JobStatus.Created) + + assert is_job_running(JobStatus.Processing) + assert is_job_running(JobStatus.ReadyForProcessing) + assert not is_job_running(JobStatus.Completed) + assert not is_job_running(JobStatus.Failed) diff --git a/tests/test_response_type_enum.py b/tests/test_response_type_enum.py new file mode 100644 index 0000000..5592817 --- /dev/null +++ b/tests/test_response_type_enum.py @@ -0,0 +1,174 @@ +""" +Unit tests for response_type_enum module. + +Copyright (c) 2016 - present Syncleus, Inc. +Copyright (c) 2016 - present CleverThis, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest + +from cleverswarm_python_client.libs.response_type_enum import ResponseTypeAPI + + +class TestResponseTypeAPI: + """Test cases for ResponseTypeAPI enum.""" + + def test_enum_values(self): + """Test that ResponseTypeAPI has the correct values.""" + assert ResponseTypeAPI.JsonText == "JsonText" + assert ResponseTypeAPI.JsonFile == "JsonFile" + + def test_enum_membership(self): + """Test that ResponseTypeAPI members can be checked for membership.""" + # Test enum member membership (works in all Python versions) + assert ResponseTypeAPI.JsonText in ResponseTypeAPI + assert ResponseTypeAPI.JsonFile in ResponseTypeAPI + + # Test string value membership (Python 3.12+ only) + import sys + if sys.version_info >= (3, 12): + assert "JsonText" in ResponseTypeAPI + assert "JsonFile" in ResponseTypeAPI + assert "InvalidType" not in ResponseTypeAPI + + def test_enum_iteration(self): + """Test that ResponseTypeAPI can be iterated over.""" + values = list(ResponseTypeAPI) + expected_values = ["JsonText", "JsonFile"] + assert len(values) == 2 + for value in expected_values: + assert value in values + + def test_enum_comparison(self): + """Test that ResponseTypeAPI values can be compared.""" + assert ResponseTypeAPI.JsonText == "JsonText" + assert ResponseTypeAPI.JsonFile == "JsonFile" + assert ResponseTypeAPI.JsonText != ResponseTypeAPI.JsonFile + + def test_enum_string_representation(self): + """Test string representation of ResponseTypeAPI values.""" + assert str(ResponseTypeAPI.JsonText) == "JsonText" + assert str(ResponseTypeAPI.JsonFile) == "JsonFile" + + def test_enum_repr(self): + """Test repr representation of ResponseTypeAPI values.""" + # Python 3.12+ uses a different repr format for enums + assert "ResponseTypeAPI.JsonText" in repr(ResponseTypeAPI.JsonText) + assert "ResponseTypeAPI.JsonFile" in repr(ResponseTypeAPI.JsonFile) + + def test_enum_inheritance(self): + """Test that ResponseTypeAPI inherits from StrEnum.""" + from enum import StrEnum + assert issubclass(ResponseTypeAPI, StrEnum) + + def test_enum_equality_with_string(self): + """Test that ResponseTypeAPI values are equal to their string representations.""" + assert ResponseTypeAPI.JsonText == "JsonText" + assert ResponseTypeAPI.JsonFile == "JsonFile" + + def test_enum_case_sensitivity(self): + """Test that ResponseTypeAPI values are case sensitive.""" + assert ResponseTypeAPI.JsonText != "jsontext" + assert ResponseTypeAPI.JsonFile != "jsonfile" + + def test_enum_ordering(self): + """Test that ResponseTypeAPI values can be ordered.""" + response_types = [ResponseTypeAPI.JsonFile, ResponseTypeAPI.JsonText] + sorted_types = sorted(response_types) + expected_order = ["JsonFile", "JsonText"] + assert [str(t) for t in sorted_types] == expected_order + + def test_enum_usage_in_conditionals(self): + """Test that ResponseTypeAPI can be used in conditional statements.""" + response_type = ResponseTypeAPI.JsonText + + if response_type == ResponseTypeAPI.JsonText: + assert True + else: + assert False + + def test_enum_usage_in_switch_like_statements(self): + """Test that ResponseTypeAPI can be used in switch-like statements.""" + def process_response_type(response_type): + if response_type == ResponseTypeAPI.JsonText: + return "processing_json_text" + elif response_type == ResponseTypeAPI.JsonFile: + return "processing_json_file" + else: + return "unknown_type" + + assert process_response_type(ResponseTypeAPI.JsonText) == "processing_json_text" + assert process_response_type(ResponseTypeAPI.JsonFile) == "processing_json_file" + + def test_enum_unique_values(self): + """Test that ResponseTypeAPI has unique values.""" + values = [member.value for member in ResponseTypeAPI] + assert len(values) == len(set(values)) + + def test_enum_serialization(self): + """Test that ResponseTypeAPI values can be serialized.""" + import json + + # Test JSON serialization + data = {"response_type": ResponseTypeAPI.JsonText} + json_str = json.dumps(data) + parsed_data = json.loads(json_str) + assert parsed_data["response_type"] == "JsonText" + + def test_enum_deserialization(self): + """Test that ResponseTypeAPI values can be deserialized.""" + # Test that string values can be converted back to enum + json_text = ResponseTypeAPI("JsonText") + json_file = ResponseTypeAPI("JsonFile") + + assert json_text == ResponseTypeAPI.JsonText + assert json_file == ResponseTypeAPI.JsonFile + + def test_enum_invalid_value(self): + """Test that invalid values raise ValueError.""" + with pytest.raises(ValueError): + ResponseTypeAPI("InvalidValue") + + def test_enum_hashable(self): + """Test that ResponseTypeAPI values are hashable.""" + # Test that enum values can be used as dictionary keys + response_map = { + ResponseTypeAPI.JsonText: "text_response", + ResponseTypeAPI.JsonFile: "file_response" + } + + assert response_map[ResponseTypeAPI.JsonText] == "text_response" + assert response_map[ResponseTypeAPI.JsonFile] == "file_response" + + def test_enum_in_sets(self): + """Test that ResponseTypeAPI values can be used in sets.""" + response_set = {ResponseTypeAPI.JsonText, ResponseTypeAPI.JsonFile} + assert len(response_set) == 2 + assert ResponseTypeAPI.JsonText in response_set + assert ResponseTypeAPI.JsonFile in response_set + + def test_enum_comparison_operators(self): + """Test comparison operators with ResponseTypeAPI values.""" + # Test equality + assert ResponseTypeAPI.JsonText == ResponseTypeAPI.JsonText + assert ResponseTypeAPI.JsonFile == ResponseTypeAPI.JsonFile + + # Test inequality + assert ResponseTypeAPI.JsonText != ResponseTypeAPI.JsonFile + + # Test with strings + assert ResponseTypeAPI.JsonText == "JsonText" + assert ResponseTypeAPI.JsonFile == "JsonFile" + assert ResponseTypeAPI.JsonText != "JsonFile" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6a1c43f --- /dev/null +++ b/tox.ini @@ -0,0 +1,60 @@ +[tox] +envlist = py38, py39, py310, py311, py312, py313, lint, type, coverage +isolated_build = True +skip_missing_interpreters = true + +[testenv] +deps = + pytest>=6.0.0 + pytest-cov>=2.12.0 + pytest-html>=4.1.1 + requests>=2.32.4 + httmock>=1.3.0 +commands = + pytest {posargs:tests} --html=unit_tests_report.html --self-contained-html --junitxml=unit_tests_report.xml --cov=cleverswarm_python_client + +[testenv:lint] +deps = + flake8>=3.9.2 + black>=25.9.0 + isort>=5.9.1 +commands = + flake8 --append-config=tox.ini src/cleverswarm_python_client + black -t py311 -t py312 -t py313 --check src/cleverswarm_python_client + isort --check-only --profile black src/cleverswarm_python_client + +[testenv:type] +deps = + mypy>=0.812 +commands = + mypy src/cleverswarm_python_client + +[testenv:coverage] +deps = + pytest>=6.0.0 + pytest-cov>=2.12.0 + coverage>=5.5 + requests>=2.32.4 + httmock>=1.3.0 +commands = + pytest {posargs:tests} --cov=cleverswarm_python_client --cov-report=term --cov-report=xml:coverage_report.xml --cov-report=html + coverage report --fail-under=85 + +[testenv:format] +deps = + black>=25.9.0 + isort>=5.9.1 +commands = + black -t py311 -t py312 -t py313 src/cleverswarm_python_client tests + isort --profile black src/cleverswarm_python_client tests + +[flake8] +max-line-length = 120 +exclude = .tox,*.egg,build,data +select = E,W,F +ignore = E203, W503 + +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_*