feat: Initial commit of the SDK at version v0.0.1
This commit is contained in:
CoreRasurae
@@ -0,0 +1,496 @@
|
||||
"""
|
||||
CleverSwarm - text to knowledge graph extraction benchmark client.
|
||||
|
||||
Copyright (c) 2016 - present Syncleus, Inc.
|
||||
Copyright (c) 2016 - present CleverThis, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set
|
||||
|
||||
from cleverswarm_python_client.libs.cleverswarm_client import CleverSwarmClient
|
||||
from cleverswarm_python_client.libs.evaluation.detailed_eval import EvaluateResults
|
||||
from cleverswarm_python_client.libs.exceptions import (
|
||||
ClientException,
|
||||
JobNotFoundException,
|
||||
)
|
||||
from cleverswarm_python_client.libs.job_enums import JobStatus
|
||||
|
||||
logger = logging.getLogger("Benchmark-client main")
|
||||
|
||||
|
||||
class BenchmarkCLI(object):
|
||||
def __init__(
|
||||
self,
|
||||
base_path: str,
|
||||
base_url: str = "http://localhost:8000/api/v0/",
|
||||
detailed_metrics: bool = False,
|
||||
username: str = None,
|
||||
token: str = None,
|
||||
):
|
||||
self._all_ontologies = {
|
||||
1: "1_university",
|
||||
2: "2_musicalwork",
|
||||
3: "3_airport",
|
||||
4: "4_building",
|
||||
5: "5_athlete",
|
||||
6: "6_politician",
|
||||
7: "7_company",
|
||||
8: "8_celestialbody",
|
||||
9: "9_astronaut",
|
||||
10: "10_comicscharacter",
|
||||
11: "11_meanoftransportation",
|
||||
12: "12_monument",
|
||||
13: "13_food",
|
||||
14: "14_writtenwork",
|
||||
15: "15_sportsteam",
|
||||
16: "16_city",
|
||||
17: "17_artist",
|
||||
18: "18_scientist",
|
||||
19: "19_film",
|
||||
20: "20_simple",
|
||||
21: "21_complex",
|
||||
}
|
||||
|
||||
self._base_url = base_url
|
||||
self._base_path: Path = Path(base_path).resolve()
|
||||
self._benchmark_path = "benchmark_data"
|
||||
self._benchmark_output = self._benchmark_path + "/local_metrics"
|
||||
self._input_template = "test_new/ont_$$onto$$_test.jsonl"
|
||||
self._onto_template = (
|
||||
"ontologies_enriched/$$onto$$_ontology_with_descriptions.json"
|
||||
)
|
||||
self._ground_truth_template = "ground_truth_new/ont_$$onto$$_ground_truth.jsonl"
|
||||
self._output_template = (
|
||||
self._benchmark_path + "/outputs_new/$$onto$$_output.jsonl"
|
||||
)
|
||||
self._detailed_metrics = detailed_metrics
|
||||
|
||||
self._client: CleverSwarmClient = CleverSwarmClient(
|
||||
self._base_url, username=username, token=token
|
||||
)
|
||||
|
||||
def find_invalid_ontologies(self, ontologies_ids: List[int]) -> List[int]:
|
||||
"""
|
||||
Find ontologies IDs that are unknown to the benchmark dataset.
|
||||
:param ontologies_ids: the ids to be verified
|
||||
:return: the list of invalid ids, or empty if no invalid
|
||||
"""
|
||||
invalids: List[int] = []
|
||||
for ontology_id in ontologies_ids:
|
||||
if ontology_id not in self._all_ontologies.keys():
|
||||
invalids.append(ontology_id)
|
||||
return invalids
|
||||
|
||||
def create_benchmark(self, ontologies_ids: List[int]) -> str:
|
||||
"""
|
||||
Creates a new benchmark job in the CleverThis REST API server.
|
||||
:param ontologies_ids: the ontologies ids of the ontologies that are to be benchmarked.
|
||||
:return: the job_id of the created job, or None if no job was created
|
||||
"""
|
||||
if len(ontologies_ids) == 0:
|
||||
return None
|
||||
|
||||
ontologies = [
|
||||
self._all_ontologies[ontology_id] for ontology_id in ontologies_ids
|
||||
]
|
||||
for idx in range(len(ontologies)):
|
||||
ontology_name: str = ontologies[idx]
|
||||
test_file: Path = self._base_path / self._input_template.replace(
|
||||
"$$onto$$", ontology_name
|
||||
)
|
||||
ontology_file: Path = self._base_path / self._onto_template.replace(
|
||||
"$$onto$$", ontology_name
|
||||
)
|
||||
ground_truth_file: Path = (
|
||||
self._base_path
|
||||
/ self._ground_truth_template.replace("$$onto$$", ontology_name)
|
||||
)
|
||||
last_fileset = idx == len(ontologies) - 1
|
||||
if idx == 0:
|
||||
job_id = self._client.create_benchmark_job(
|
||||
test_file, ontology_file, ground_truth_file, last_fileset
|
||||
)
|
||||
else:
|
||||
self._client.append_to_benchmark_job(
|
||||
job_id, test_file, ontology_file, ground_truth_file, last_fileset
|
||||
)
|
||||
|
||||
return job_id
|
||||
|
||||
def poll_job_to_completion(self, job_id: str) -> None:
|
||||
"""
|
||||
Polls a job by its job id and waits for the job completion in the server.
|
||||
It throws an exception if the job failed or is in a state that cannot be executed.
|
||||
:param job_id: the job id of the job to be executed.
|
||||
"""
|
||||
status: JobStatus = self._client.get_job_status(job_id)
|
||||
if status == JobStatus.Failed:
|
||||
raise ClientException(f"Job with ID: {job_id} finished without completing.")
|
||||
if status == JobStatus.Completed:
|
||||
return
|
||||
if status != JobStatus.ReadyForProcessing and status != JobStatus.Processing:
|
||||
raise ClientException(
|
||||
f"Job with ID: {job_id} is not processing, nor is it ready for processing."
|
||||
)
|
||||
final_status: JobStatus = self._client.poll_job_ready_or_failed(job_id)
|
||||
if final_status != JobStatus.Completed:
|
||||
raise ClientException(f"Job with ID: {job_id} finished without completing.")
|
||||
|
||||
def download_job_results(self, job_id: str, ontologies_ids: List[int]) -> None:
|
||||
"""
|
||||
Download the results from the server, for the specified job and the specified ontologies_ids.
|
||||
:param job_id: the job_id from which the results are to be retrieved
|
||||
:param ontologies_ids: the ontologies_ids that are to be retrieved
|
||||
"""
|
||||
ontologies = [
|
||||
self._all_ontologies[ontology_id] for ontology_id in ontologies_ids
|
||||
]
|
||||
|
||||
output_filenames = []
|
||||
for idx in range(len(ontologies)):
|
||||
ontology_name: str = ontologies[idx]
|
||||
output_file: Path = self._base_path / self._output_template.replace(
|
||||
"$$onto$$", ontology_name
|
||||
)
|
||||
output_filenames.append(output_file)
|
||||
|
||||
self._client.retrieve_benchmark_files(job_id, output_filenames)
|
||||
print(
|
||||
f"Benchmark job with ID: {job_id}, produced the following output files: {output_filenames}"
|
||||
)
|
||||
|
||||
def print_ontologies_ids(self) -> None:
|
||||
"""
|
||||
Prints the list of all known benchmark ontologies and their IDs.
|
||||
"""
|
||||
print("ID, Name")
|
||||
for item in self._all_ontologies.items():
|
||||
print("{:>2}, {}".format(item[0], item[1]))
|
||||
|
||||
def list_server_jobs(self) -> Dict:
|
||||
"""
|
||||
List all benchmark jobs in the server that are owned by the current logged in user.
|
||||
:return: JSON dictionary with job listing and respective states.
|
||||
"""
|
||||
json_data = self._client.get_jobs_list()
|
||||
return json_data
|
||||
|
||||
def extract_server_job_ontologies(self, job_id: str) -> List[int]:
|
||||
"""
|
||||
Obtains the ontologies ids from a given server job, identified by its ID.
|
||||
:param job_id: the job id of the server job to be consulted.
|
||||
:return: the ontologies ids found in the specified benchmark server job
|
||||
"""
|
||||
found_job = self._client.get_job_details(job_id)
|
||||
|
||||
if found_job is None:
|
||||
raise JobNotFoundException(
|
||||
f"Could not find job with ID: {job_id} in the server"
|
||||
)
|
||||
|
||||
results: List[int] = []
|
||||
for source in found_job["ontology_sources"]:
|
||||
idx_str = source.split("_")[0]
|
||||
results.append(int(idx_str))
|
||||
|
||||
return results
|
||||
|
||||
def print_server_jobs(self, detailed: bool = False) -> None:
|
||||
"""
|
||||
Prints a summarized or detailed status of the benchmark jobs in the server.
|
||||
"""
|
||||
jobs = self.list_server_jobs()
|
||||
for job in jobs:
|
||||
print("Job ID: {}".format(job["id"]))
|
||||
print("Created at: {}".format(job["created"]))
|
||||
print("Type: {}".format(job["type"]))
|
||||
print("Status: {}".format(job["status"]))
|
||||
print("Retries: {}".format(job["retries_count"]))
|
||||
print("Input file type: {}".format((job["file_type"])))
|
||||
if detailed:
|
||||
print("Unstructured text input files:")
|
||||
for source in job["unstructured_sources"]:
|
||||
print(f" {source}")
|
||||
print("Ontology input files:")
|
||||
for source in job["ontology_sources"]:
|
||||
print(f" {source}")
|
||||
print("Ground-truth input files:")
|
||||
for source in job["ground_truth_sources"]:
|
||||
print(f" {source}")
|
||||
print("----------------------------")
|
||||
|
||||
def delete_server_job(self, job_id: str) -> None:
|
||||
"""
|
||||
Deletes a given job by its ID. A job can only be deleted if it is in a state that allows it.
|
||||
:param job_id: the server job id
|
||||
"""
|
||||
del_status = self._client.delete_job(job_id)
|
||||
if not del_status:
|
||||
print("Failed to delete Job")
|
||||
else:
|
||||
print("Job was successfully deleted")
|
||||
|
||||
def evaluate_results_locally(self, ontologies_ids: List[int]) -> Path:
|
||||
"""
|
||||
Evaluates the extraction data obtained from the server and in the local machine to obtain the benchmark data.
|
||||
The scores can be either detailed or summarized.
|
||||
:param ontologies_ids: the ids of the ontologies to be benchmarked, it can include already downloaded ids
|
||||
from previous benchmark jobs, for which the data has already been downloaded.
|
||||
"""
|
||||
ontologies = [
|
||||
self._all_ontologies[ontology_id] for ontology_id in ontologies_ids
|
||||
]
|
||||
json_dict: Dict = {
|
||||
"onto_list": ontologies,
|
||||
"path_patterns": {
|
||||
"sys": str(self._base_path / self._output_template),
|
||||
"gt": str(self._base_path / self._ground_truth_template),
|
||||
"onto": str(self._base_path / self._onto_template),
|
||||
"output": str(
|
||||
self._base_path
|
||||
/ self._benchmark_output
|
||||
/ "ont_$$onto$$_eval_results.jsonl"
|
||||
),
|
||||
},
|
||||
"overall_output": str(
|
||||
self._base_path / self._benchmark_output / "avg_eval_results.jsonl"
|
||||
),
|
||||
}
|
||||
|
||||
EvaluateResults(json_dict, self._detailed_metrics)
|
||||
return self._base_path / self._benchmark_output
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the benchmark client console script."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CleverSwarm benchmark client for REST API."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ontologies_ids",
|
||||
nargs="+",
|
||||
type=int,
|
||||
help="List of ontologies IDs to be processed",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--local_metrics_ontologies_ids",
|
||||
nargs="+",
|
||||
type=int,
|
||||
help="List of ontologies IDs to be benchmarked (can include locally cached test ontologies results).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bench_data_prefix",
|
||||
type=str,
|
||||
default="./data/dbpedia",
|
||||
help="Path prefix to reach the common benchmark folder structure.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--username", type=str, help="Username for logging in into server"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
type=str,
|
||||
help="Valid Bearer Token value to avoid the need for authentication",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server_url",
|
||||
type=str,
|
||||
default="http://localhost:8000/api/v0/",
|
||||
help="REST server URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--action",
|
||||
choices=[
|
||||
"create-and-exit",
|
||||
"delete-and-exit",
|
||||
"metrics-and-exit",
|
||||
"download-metrics-and-exit",
|
||||
"list-benchmark-jobs-and-exit",
|
||||
"list-ontologies-ids-and-exit",
|
||||
"create-download-metrics-delete",
|
||||
"create-download-metrics",
|
||||
],
|
||||
type=str,
|
||||
help="Define action to be performed",
|
||||
default="create-download-metrics-delete",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job_id",
|
||||
type=str,
|
||||
help="Identify a specific job. To be used with create-and-exit, delete-and-exit, download-metrics-and-exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detailed",
|
||||
default=False,
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
create: bool = False
|
||||
poll: bool = False
|
||||
download: bool = False
|
||||
metrics: bool = False
|
||||
delete: bool = False
|
||||
list_jobs: bool = False
|
||||
list_ontologies: bool = False
|
||||
|
||||
if args.action == "create-and-exit":
|
||||
create = True
|
||||
poll = False
|
||||
if args.ontologies_ids is None:
|
||||
print("Create-and-exit action requires --ontologies_ids argument.\n")
|
||||
exit(-1)
|
||||
elif args.action == "delete-and-exit":
|
||||
delete = True
|
||||
if args.job_id is None:
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "metrics-and-exit":
|
||||
metrics = True
|
||||
if args.local_metrics_ontologies_ids is None:
|
||||
print("Metrics-and-exit action requires --ontologies_ids argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "download-metrics-and-exit":
|
||||
poll = True
|
||||
download = True
|
||||
metrics = True
|
||||
if args.job_id is None:
|
||||
print("Download-metrics-and-exit action requires --job_id argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "list-benchmark-jobs-and-exit":
|
||||
list_jobs = True
|
||||
elif args.action == "create-download-metrics-delete":
|
||||
create = True
|
||||
poll = True
|
||||
download = True
|
||||
metrics = True
|
||||
delete = True
|
||||
if args.ontologies_ids is None:
|
||||
print(
|
||||
"Create-download-metrics-delete action requires --ontologies_ids argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "create-download-metrics":
|
||||
create = True
|
||||
poll = True
|
||||
download = True
|
||||
metrics = True
|
||||
if args.ontologies_ids is None:
|
||||
print(
|
||||
"Create-download-metrics action requires --ontologies_ids argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "list-ontologies-ids-and-exit":
|
||||
list_ontologies = True
|
||||
|
||||
url: str = args.server_url
|
||||
if not url.endswith("/"):
|
||||
url += "/"
|
||||
|
||||
ontologies_ids: List[int] = None
|
||||
if args.ontologies_ids is not None:
|
||||
ontologies_ids_set: Set[int] = set(args.ontologies_ids)
|
||||
ontologies_ids: List[int] = sorted(list(ontologies_ids_set))
|
||||
|
||||
cli: BenchmarkCLI = BenchmarkCLI(
|
||||
args.bench_data_prefix,
|
||||
url,
|
||||
args.detailed,
|
||||
username=args.username,
|
||||
token=args.token,
|
||||
)
|
||||
try:
|
||||
job_id = args.job_id
|
||||
|
||||
if ontologies_ids is not None:
|
||||
invalids = cli.find_invalid_ontologies(ontologies_ids)
|
||||
if len(invalids) > 0:
|
||||
print("Invalid IDs found in option --ontologies_ids.")
|
||||
print(f"The following ontologies ids are invalid: {invalids}\n")
|
||||
print(
|
||||
"Use --list-ontologies-ids-and-exit to find the supported benchmark ontologies ids.\n"
|
||||
)
|
||||
print("Type 'python main.py -h' for Help")
|
||||
exit(-1)
|
||||
|
||||
if args.local_metrics_ontologies_ids is not None:
|
||||
invalids = cli.find_invalid_ontologies(args.local_metrics_ontologies_ids)
|
||||
if len(invalids) > 0:
|
||||
print("Invalid IDs found in option --local_metrics_ontologies_ids.")
|
||||
print(f"The following ontologies ids are invalid: {invalids}\n")
|
||||
print(
|
||||
"Use --list-ontologies-ids-and-exit to find the supported benchmark ontologies ids.\n"
|
||||
)
|
||||
print("Type 'python main.py -h' for Help")
|
||||
exit(-1)
|
||||
|
||||
if list_ontologies:
|
||||
cli.print_ontologies_ids()
|
||||
|
||||
if create:
|
||||
job_id: str = cli.create_benchmark(ontologies_ids)
|
||||
print(f"Successfully created benchmark job with ID: {job_id}.")
|
||||
|
||||
if poll:
|
||||
cli.poll_job_to_completion(job_id)
|
||||
|
||||
if download:
|
||||
if not create or ontologies_ids is None:
|
||||
ontologies_ids = cli.extract_server_job_ontologies(job_id)
|
||||
cli.download_job_results(job_id, ontologies_ids)
|
||||
|
||||
if metrics:
|
||||
local_ontologies_ids: Set = set()
|
||||
if ontologies_ids is not None and create:
|
||||
local_ontologies_ids.update(ontologies_ids)
|
||||
elif job_id is not None:
|
||||
print("Checking ontologies present in Job ID: {job_id}.")
|
||||
local_ontologies_ids.update(cli.extract_server_job_ontologies(job_id))
|
||||
if args.local_metrics_ontologies_ids is not None:
|
||||
local_ontologies_ids.update(args.local_metrics_ontologies_ids)
|
||||
|
||||
local_ontologies_ids: List[int] = sorted(list(local_ontologies_ids))
|
||||
print(
|
||||
"Considering ontologies IDs: {} for local metrics".format(
|
||||
local_ontologies_ids
|
||||
)
|
||||
)
|
||||
output_folder_path = cli.evaluate_results_locally(local_ontologies_ids)
|
||||
print(f"Results scores written to: {str(output_folder_path)}")
|
||||
|
||||
if delete:
|
||||
cli.delete_server_job(job_id)
|
||||
|
||||
if list_jobs:
|
||||
cli.print_server_jobs(args.detailed)
|
||||
except ClientException as error:
|
||||
print(str(error))
|
||||
exit(-1)
|
||||
except Exception as error:
|
||||
logger.error(f"Unexpected error occurred: {error}", exc_info=error)
|
||||
print("Unexpected error occurred, check the logs for more details")
|
||||
exit(-1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,409 @@
|
||||
"""
|
||||
CleverSwarm - unstructured text to knowledge graph triplets extractor client.
|
||||
|
||||
Copyright (c) 2016 - present Syncleus, Inc.
|
||||
Copyright (c) 2016 - present CleverThis, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from cleverswarm_python_client.libs.cleverswarm_client import CleverSwarmClient
|
||||
from cleverswarm_python_client.libs.exceptions import ClientException
|
||||
from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI
|
||||
from cleverswarm_python_client.libs.job_enums import JobStatus, JobType
|
||||
|
||||
logger = logging.getLogger("Text-to-KG-client main")
|
||||
|
||||
|
||||
class TextoToKGCLI(object):
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
detailed: bool = False,
|
||||
input_prefix: str = None,
|
||||
output_prefix: str = None,
|
||||
username: str = None,
|
||||
token: str = None,
|
||||
):
|
||||
|
||||
self._input_prefix: Path = Path(input_prefix) if input_prefix else Path("../..")
|
||||
self._output_prefix: Path = (
|
||||
Path(output_prefix) if output_prefix else Path("../..")
|
||||
)
|
||||
self._username: str = username
|
||||
|
||||
self._input_prefix = self._input_prefix.resolve()
|
||||
self._output_prefix = self._output_prefix.resolve()
|
||||
|
||||
if not self._input_prefix.exists() or not self._input_prefix.is_dir():
|
||||
raise ClientException(
|
||||
f"Input prefix does not exist or is not a folder: {input_prefix}"
|
||||
)
|
||||
|
||||
self._output_prefix.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not self._output_prefix.exists() or not self._output_prefix.is_dir():
|
||||
raise ClientException(
|
||||
f"Input prefix could not be created or is not a folder: {output_prefix}"
|
||||
)
|
||||
|
||||
self._client: CleverSwarmClient = CleverSwarmClient(base_url, token, username)
|
||||
|
||||
self._detailed = detailed
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
input_text: str,
|
||||
ontology_json: str,
|
||||
ontology_owl: str,
|
||||
wildcards: str = None,
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
):
|
||||
input_text_path: Path = self._input_prefix / input_text
|
||||
ontology_json_path: Path = self._input_prefix / ontology_json
|
||||
ontology_owl_path: Path = self._input_prefix / ontology_owl
|
||||
wildcards_path: Path = Path(wildcards) if wildcards else None
|
||||
|
||||
if not input_text_path.exists() or not input_text_path.is_file():
|
||||
raise ClientException(
|
||||
f"Unstructured input text does not exist or is not a file: {str(input_text_path)}"
|
||||
)
|
||||
|
||||
if not ontology_json_path.exists() or not ontology_json_path.is_file():
|
||||
raise ClientException(
|
||||
f"Input ontology JSON file does not exist or is not a file: {str(ontology_json_path)}"
|
||||
)
|
||||
|
||||
if not ontology_owl_path.exists() or not ontology_owl_path.is_file():
|
||||
raise ClientException(
|
||||
f"Input ontology OWL file does not exist or is not a file: {str(ontology_owl_path)}"
|
||||
)
|
||||
|
||||
wildcards_job: bool = False
|
||||
if wildcards_path is not None:
|
||||
wildcards_job = True
|
||||
if not wildcards_path.exists() or not wildcards_path.is_file():
|
||||
raise ClientException(
|
||||
f"Input wildcards file does not exist or is not a file: {str(wildcards_path)}"
|
||||
)
|
||||
|
||||
if wildcards_job:
|
||||
job_id: str = self._client.create_unstructured_to_kg_wildcards_job(
|
||||
input_text_path,
|
||||
ontology_json_path,
|
||||
ontology_owl_path,
|
||||
wildcards_path,
|
||||
force_filetype=force_filetype,
|
||||
)
|
||||
else:
|
||||
job_id: str = self._client.create_unstructured_to_kg_job(
|
||||
input_text_path,
|
||||
ontology_json_path,
|
||||
ontology_owl_path,
|
||||
force_filetype=force_filetype,
|
||||
)
|
||||
|
||||
return job_id
|
||||
|
||||
def poll_job_to_completion(self, job_id: str) -> None:
|
||||
"""
|
||||
Polls a job by its job id and waits for the job completion in the server.
|
||||
It throws an exception if the job failed or is in a state that cannot be executed.
|
||||
:param job_id: the job id of the job to be executed.
|
||||
"""
|
||||
status: JobStatus = self._client.get_job_status(job_id)
|
||||
if status == JobStatus.Failed:
|
||||
raise ClientException(f"Job with ID: {job_id} finished without completing.")
|
||||
if status == JobStatus.Completed:
|
||||
return
|
||||
if status != JobStatus.ReadyForProcessing and status != JobStatus.Processing:
|
||||
raise ClientException(
|
||||
f"Job with ID: {job_id} is not processing, nor is it ready for processing."
|
||||
)
|
||||
final_status: JobStatus = self._client.poll_job_ready_or_failed(job_id)
|
||||
if final_status != JobStatus.Completed:
|
||||
raise ClientException(f"Job with ID: {job_id} finished without completing.")
|
||||
|
||||
def download_job_result(self, job_id: str) -> None:
|
||||
details: Dict[str, Any] = self._client.get_job_details(job_id)
|
||||
|
||||
if len(details["unstructured_sources"]) == 0:
|
||||
raise ClientException(
|
||||
f"Unstructured to KG job: {job_id} is not valid. It has no input files."
|
||||
)
|
||||
if len(details["unstructured_sources"]) > 1:
|
||||
raise ClientException(
|
||||
f"Unstructured to KG job: {job_id} is not valid. It has more than one input files."
|
||||
)
|
||||
input_filename: str = details["unstructured_sources"][0]
|
||||
output_filename: str = input_filename + ".json"
|
||||
output_filepath: Path = self._output_prefix / output_filename
|
||||
self._client.retrieve_unstructured_to_kg_files(job_id, output_filepath)
|
||||
if self._detailed:
|
||||
print("Job result was downloaded to: {}".format(str(output_filepath)))
|
||||
|
||||
def delete_server_job(self, job_id: str) -> None:
|
||||
"""
|
||||
Deletes a given job by its ID. A job can only be deleted if it is in a state that allows it.
|
||||
:param job_id: the server job id
|
||||
"""
|
||||
del_status = self._client.delete_job(job_id)
|
||||
if not del_status:
|
||||
print("Failed to delete Job")
|
||||
else:
|
||||
print("Job was successfully deleted")
|
||||
|
||||
def list_server_jobs(self) -> Dict:
|
||||
"""
|
||||
List all text to KG extraction jobs in the server that are owned by the current logged-in user.
|
||||
:return: JSON dictionary with job listing and respective states.
|
||||
"""
|
||||
json_data = self._client.get_jobs_list(is_benchmark=False)
|
||||
return json_data
|
||||
|
||||
def print_server_jobs(self, is_detailed: bool = False) -> None:
|
||||
"""
|
||||
Prints a summarized or detailed status of the text to KG extraction jobs in the server.
|
||||
"""
|
||||
jobs = self.list_server_jobs()
|
||||
for job in jobs:
|
||||
print("Job ID: {}".format(job["id"]))
|
||||
print("Created at: {}".format(job["created"]))
|
||||
print("Type: {}".format(job["type"]))
|
||||
print("Status: {}".format(job["status"]))
|
||||
print("Retries: {}".format(job["retries_count"]))
|
||||
print("Input file type: {}".format((job["file_type"])))
|
||||
if is_detailed:
|
||||
print("Unstructured text input files:")
|
||||
for source in job["unstructured_sources"]:
|
||||
print(f" {source}")
|
||||
print("Ontology JSON input files:")
|
||||
for source in job["ontology_sources"]:
|
||||
print(f" {source}")
|
||||
print("Ontology OWL/XML input files:")
|
||||
for source in job["ontology_spec_sources"]:
|
||||
print(f" {source}")
|
||||
if JobType(job["type"]) == JobType.UnstructuredWithWildcards:
|
||||
print("Wildcards input files:")
|
||||
for source in job["wildcards_sources"]:
|
||||
print(f" {source}")
|
||||
print("----------------------------")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the text-to-KG client console script."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="CleverSwarm unstructured text to Knowledge Graph triplets client for REST API."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input_prefix",
|
||||
type=str,
|
||||
default=".",
|
||||
help="Path prefix for the common input folder structure.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_prefix", type=str, default=".", help="Path to the output folder."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--username", type=str, help="Username for logging in into server"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
type=str,
|
||||
help="Valid Bearer Token value to avoid the need for authentication",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server_url",
|
||||
type=str,
|
||||
default="http://localhost:8000/api/v0/",
|
||||
help="REST server URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--action",
|
||||
choices=[
|
||||
"create-and-exit",
|
||||
"delete-and-exit",
|
||||
"download-kg-and-exit",
|
||||
"list-kg-jobs-and-exit",
|
||||
"create-download-kg-delete",
|
||||
"create-download-kg",
|
||||
],
|
||||
type=str,
|
||||
help="Define action to be performed",
|
||||
default="create-download-kg-delete",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job_id",
|
||||
type=str,
|
||||
help="Identify a specific job. To be used with create-and-exit, delete-and-exit, download-metrics-and-exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--unstructured_text",
|
||||
type=str,
|
||||
help="Unstructured text input file for the Knowledge Graph triplet extraction",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ontology_json",
|
||||
type=str,
|
||||
help="Enriched ontology file with descriptions in JSON format",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ontology_owl",
|
||||
type=str,
|
||||
help="OWL ontology file in XML format",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wildcards",
|
||||
type=str,
|
||||
help="Wildcards ontology file. If specified, a wildcards filtering job will be created instead "
|
||||
"of a regular KG extraction.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--detailed",
|
||||
default=False,
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
create = False
|
||||
poll = False
|
||||
download = False
|
||||
delete = False
|
||||
list_jobs = False
|
||||
|
||||
if args.action == "create-and-exit":
|
||||
create = True
|
||||
poll = False
|
||||
if args.unstructured_text is None:
|
||||
print("create-and-exit action requires --unstructured_text argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print("create-and-exit action requires --ontology_json argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print("create-and-exit action requires --ontology_owl argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "delete-and-exit":
|
||||
delete = True
|
||||
if args.job_id is None:
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "download-kg-and-exit":
|
||||
poll = True
|
||||
download = True
|
||||
if args.job_id is None:
|
||||
print("download-kg-and-exit action requires --job_id argument.\n")
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "list-kg-jobs-and-exit":
|
||||
list_jobs = True
|
||||
elif args.action == "create-download-kg-delete":
|
||||
create = True
|
||||
poll = True
|
||||
download = True
|
||||
delete = True
|
||||
if args.unstructured_text is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --unstructured_text argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_json argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_owl argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
elif args.action == "create-download-kg":
|
||||
create = True
|
||||
poll = True
|
||||
download = True
|
||||
if args.unstructured_text is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --unstructured_text argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_json is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_json argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
if args.ontology_owl is None:
|
||||
print(
|
||||
"create-download-kg-delete action requires --ontology_owl argument.\n"
|
||||
)
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
|
||||
url: str = args.server_url
|
||||
if not url.endswith("/"):
|
||||
url += "/"
|
||||
|
||||
try:
|
||||
job_id = args.job_id
|
||||
cli = TextoToKGCLI(
|
||||
base_url=url,
|
||||
detailed=args.detailed,
|
||||
input_prefix=args.input_prefix,
|
||||
output_prefix=args.output_prefix,
|
||||
username=args.username,
|
||||
token=args.token,
|
||||
)
|
||||
if create:
|
||||
job_id = cli.create_job(
|
||||
args.unstructured_text,
|
||||
args.ontology_json,
|
||||
args.ontology_owl,
|
||||
args.wildcards,
|
||||
)
|
||||
|
||||
if poll:
|
||||
cli.poll_job_to_completion(job_id)
|
||||
|
||||
if download:
|
||||
cli.download_job_result(job_id)
|
||||
|
||||
if delete:
|
||||
cli.delete_server_job(job_id)
|
||||
|
||||
if list_jobs:
|
||||
cli.print_server_jobs(args.detailed)
|
||||
|
||||
except ClientException as msg:
|
||||
print(str(msg))
|
||||
exit(-1)
|
||||
except Exception as error:
|
||||
logger.error(f"Unexpected error occurred: {error}", exc_info=error)
|
||||
print("Unexpected error occurred, check the logs for more details")
|
||||
exit(-1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,817 @@
|
||||
"""
|
||||
Module for interfacing with CleverSwarm REST API server
|
||||
|
||||
Copyright (c) 2016 - present Syncleus, Inc.
|
||||
Copyright (c) 2016 - present CleverThis, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from getpass import getpass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Self
|
||||
|
||||
import requests
|
||||
|
||||
from cleverswarm_python_client.libs.exceptions import (
|
||||
BaseUrlMissingException,
|
||||
ClientException,
|
||||
ConnectionErrorException,
|
||||
ConnectionTimeoutException,
|
||||
InvalidFileException,
|
||||
InvalidUsernameOrPasswordException,
|
||||
JobNotFoundException,
|
||||
RequestErrorException,
|
||||
UnauthorizedException,
|
||||
UnexpectedConditionException,
|
||||
)
|
||||
from cleverswarm_python_client.libs.file_type_enum import FileTypeAPI
|
||||
from cleverswarm_python_client.libs.job_enums import JobStatus, JobType
|
||||
from cleverswarm_python_client.libs.response_type_enum import ResponseTypeAPI
|
||||
|
||||
logger = logging.getLogger("Benchmark-client")
|
||||
|
||||
|
||||
class CleverSwarmClient(object):
|
||||
"""
|
||||
CleverSwarmClient is a class that provides an abstraction for interacting with a CleverSwarm REST API server.
|
||||
"""
|
||||
|
||||
_max_retries: int = 2
|
||||
_timeout: int = 10
|
||||
_base_url: str = None
|
||||
_token: str = None
|
||||
_username: str = None
|
||||
|
||||
def __init__(self, base_url: str, token: str = None, username: str = None):
|
||||
"""
|
||||
Instantiates a CleverSwarmClient instance.
|
||||
:param base_url: the base_url of the REST API server
|
||||
:param token: a token if already available, possibly avoiding a login step if the token is still valid
|
||||
:param username: the username for which the login should be performed, if not specified, it will be asked
|
||||
"""
|
||||
self._base_url = base_url
|
||||
self._token = token
|
||||
self._username = username
|
||||
self._triplets_template = "{}_output.jsonl"
|
||||
|
||||
def _generic_http_request_executor(
|
||||
self, blogic_method_wrapper, req_name: str = "request", is_login: bool = False
|
||||
):
|
||||
"""
|
||||
Helper method to execute
|
||||
:param blogic_method_wrapper: the wrapper function to the actual business logic
|
||||
:param req_name: a text that summarizes the request to be invoked
|
||||
:param is_login: True, indicates it is a login request
|
||||
:return: the blogic_method_wrapper return value
|
||||
"""
|
||||
retries = 0
|
||||
relogin = False
|
||||
while retries <= self._max_retries:
|
||||
try:
|
||||
if not is_login and relogin:
|
||||
self.update_token()
|
||||
relogin = False
|
||||
|
||||
return blogic_method_wrapper()
|
||||
except Exception as error:
|
||||
if isinstance(error, ClientException):
|
||||
if retries < self._max_retries and isinstance(
|
||||
error, UnauthorizedException
|
||||
):
|
||||
if (
|
||||
str(error).lower().strip()
|
||||
== "user provided credentials are not correct."
|
||||
):
|
||||
relogin = True
|
||||
elif retries < self._max_retries and isinstance(
|
||||
error, InvalidUsernameOrPasswordException
|
||||
):
|
||||
relogin = True
|
||||
else:
|
||||
raise error
|
||||
elif isinstance(error, requests.exceptions.HTTPError):
|
||||
logger.error(
|
||||
f"Failed to execute {req_name} with cause {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise RequestErrorException(
|
||||
"Failed process request, error from server."
|
||||
)
|
||||
elif isinstance(error, requests.exceptions.ConnectionError):
|
||||
logger.error(
|
||||
f"Failed to execute {req_name} due connection error {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise ConnectionErrorException("Failed to connect to server.")
|
||||
elif isinstance(error, requests.exceptions.ConnectTimeout):
|
||||
logger.error(
|
||||
f"Failed to execute {req_name} due connection timeout {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise ConnectionTimeoutException(
|
||||
"Communication with server timed out."
|
||||
)
|
||||
elif isinstance(error, requests.exceptions.ReadTimeout):
|
||||
logger.error(
|
||||
f"Failed to execute {req_name} due read timeout {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise ConnectionTimeoutException(
|
||||
"Communication timed out while receiving data from the server."
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to execute {req_name} due to unexpected error {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise UnexpectedConditionException(
|
||||
"Client error while processing authorization"
|
||||
)
|
||||
retries += 1
|
||||
|
||||
def update_token(self, query_username: bool = False) -> Self:
|
||||
"""
|
||||
Updates the session authorization token.
|
||||
:return: the BenchmarkClient instance
|
||||
"""
|
||||
if self._base_url is None or self._base_url.strip() == "":
|
||||
raise BaseUrlMissingException(
|
||||
"Cannot update token, because no URL base was provided."
|
||||
)
|
||||
|
||||
def func_get_updated_token():
|
||||
if self._username is None or query_username:
|
||||
self._username = input("Please enter your username: ")
|
||||
password = getpass("Please enter your password: ")
|
||||
# TODO Hash password
|
||||
data = {"username": self._username, "password": password}
|
||||
|
||||
r = requests.post(
|
||||
self._base_url + "login", data=data, timeout=self._timeout
|
||||
)
|
||||
if r.status_code == 400:
|
||||
raise InvalidUsernameOrPasswordException(
|
||||
"Invalid username or password entered."
|
||||
)
|
||||
else:
|
||||
r.raise_for_status()
|
||||
|
||||
token_data = r.json()
|
||||
if token_data["token_type"].lower() != "bearer":
|
||||
logger.error(
|
||||
"Unexpected token_type: {}".format(token_data["token_type"])
|
||||
)
|
||||
raise UnexpectedConditionException(
|
||||
"Unexpected token type on authorization"
|
||||
)
|
||||
|
||||
self._token = token_data["access_token"]
|
||||
return None
|
||||
|
||||
self._generic_http_request_executor(
|
||||
func_get_updated_token, "authorization request", True
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def get_jobs_list(self, is_benchmark: bool = True) -> List:
|
||||
"""
|
||||
Obtains the list of jobs of the specified type in the server along with their statuses.
|
||||
Login is requested if required by the server.
|
||||
:param is_benchmark: True, if only benchmark jobs are to be returned.
|
||||
False, if only regular text to KG extraction jobs are to be returned.
|
||||
:return: the list of benchmark jobs
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
def func_get_jobs_list():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
response = requests.get(
|
||||
self._base_url + "jobs", headers=headers, timeout=self._timeout
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
|
||||
test = None
|
||||
if is_benchmark:
|
||||
|
||||
def test_is_bench(x: JobType) -> bool:
|
||||
return x == JobType.Benchmark
|
||||
|
||||
test = test_is_bench
|
||||
else:
|
||||
|
||||
def test_is_not_bench(x: JobType) -> bool:
|
||||
return x != JobType.Benchmark
|
||||
|
||||
test = test_is_not_bench
|
||||
|
||||
# Filter response to only return Benchmark jobs
|
||||
results = []
|
||||
response_json = response.json()
|
||||
for result in response_json:
|
||||
if test(JobType(result["type"])):
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_get_jobs_list, "get jobs list request"
|
||||
)
|
||||
|
||||
def get_job_status(self, job_id: str) -> JobStatus:
|
||||
"""
|
||||
Obtains the job status.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the job id
|
||||
:return: the current job status in the server
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
def func_get_job_status():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
response = requests.get(
|
||||
self._base_url + "jobs/" + job_id,
|
||||
headers=headers,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif e.response.status_code == 404:
|
||||
raise JobNotFoundException(
|
||||
"Job with id: {} not found.".format(job_id)
|
||||
)
|
||||
else:
|
||||
logger.error(f"Error retrieving job details: {e}")
|
||||
raise RequestErrorException(f"Failed to retrieve job details: {e}")
|
||||
|
||||
logger.debug(f"Server responded: {response.json()}")
|
||||
return response.json()
|
||||
|
||||
status_str: str = self._generic_http_request_executor(
|
||||
func_get_job_status, "get job status request"
|
||||
)
|
||||
return JobStatus(status_str)
|
||||
|
||||
def get_job_details(self, job_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Obtains the job details including input files and status.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the job id
|
||||
:return: the job details dictionary
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
def func_get_job_details():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
response = requests.get(
|
||||
self._base_url + "jobs/details/" + job_id,
|
||||
headers=headers,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif e.response.status_code == 404:
|
||||
raise JobNotFoundException(
|
||||
"Job with id: {} not found.".format(job_id)
|
||||
)
|
||||
else:
|
||||
logger.error(f"Error retrieving job details: {e}")
|
||||
raise RequestErrorException(f"Failed to retrieve job details: {e}")
|
||||
|
||||
logger.debug(f"Server responded: {response.json()}")
|
||||
return response.json()
|
||||
|
||||
details_str = self._generic_http_request_executor(
|
||||
func_get_job_details, "get job details request"
|
||||
)
|
||||
return details_str
|
||||
|
||||
def delete_job(self, job_id: str):
|
||||
"""
|
||||
Deletes a given job by its job_id.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the job id of the job to be deleted
|
||||
:return: True, if the job was successfully deleted, False, otherwise.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
def func_delete_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
response = requests.delete(
|
||||
self._base_url + "jobs/" + job_id,
|
||||
headers=headers,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(response.json()["detail"])
|
||||
elif response.status_code == 404:
|
||||
raise JobNotFoundException(f"Job with ID: {job_id} was not found.")
|
||||
elif response.status_code == 412:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_delete_job, "delete job request"
|
||||
)
|
||||
|
||||
def retry_job(self, job_id: str) -> JobStatus:
|
||||
"""
|
||||
Retry a given job specified by its job_id. Only jobs in Failed state can be retried.
|
||||
:param job_id: the job id of the job to be retried
|
||||
:return: the new job status and configuration
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
def func_retry_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
response = requests.put(
|
||||
self._base_url + "jobs/" + job_id,
|
||||
headers=headers,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif response.status_code == 404:
|
||||
raise JobNotFoundException(f"Job with ID: {job_id} was not found.")
|
||||
elif response.status_code == 412:
|
||||
raise ClientException(
|
||||
"Cannot retry job because it is not in Failed state."
|
||||
)
|
||||
|
||||
logger.debug(f"Server responded: {response.json()}")
|
||||
return response.json()
|
||||
|
||||
return self._generic_http_request_executor(func_retry_job, "retry job request")
|
||||
|
||||
def poll_job_ready_or_failed(self, job_id: str) -> JobStatus:
|
||||
"""
|
||||
Polls a given server job and keeps polling until the job either completes, or fails.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the job id of the server job to be polled
|
||||
:return: the final status of the job at the end of the poll
|
||||
"""
|
||||
status: JobStatus = self.get_job_status(job_id)
|
||||
while status != JobStatus.Failed and status != JobStatus.Completed:
|
||||
logger.info(f"Polling - Current status is: {status}")
|
||||
time.sleep(20)
|
||||
status = self.get_job_status(job_id)
|
||||
logger.info(f"Polling - Current status is: {status}")
|
||||
return status
|
||||
|
||||
def create_benchmark_job(
|
||||
self,
|
||||
test_file: Path,
|
||||
ontology_file: Path,
|
||||
ground_truth_file: Path,
|
||||
last_fileset=True,
|
||||
):
|
||||
"""
|
||||
Creates a benchmark job in the server and uploads the files required.
|
||||
Login is requested if required by the server.
|
||||
If the upload is successful, it returns a job ID.
|
||||
:param test_file: a valid file path containing test cases for benchmark
|
||||
:param ontology_file: a valid file containing an ontology in JSON format.
|
||||
:param ground_truth_file: a valid file path containing ground truth data.
|
||||
:param last_fileset: True, indicates this is the last file set, so the job can be placed in the ready queue.
|
||||
:return: the job ID where the files were placed.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
if not test_file.exists() or not test_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Input test file does not exist or is not a file: {str(test_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology file does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ground_truth_file.exists() or not ground_truth_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ground truth file does not exist or is not a file: {str(ground_truth_file)}"
|
||||
)
|
||||
|
||||
def func_create_benchmark_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"ready_for_processing": last_fileset}
|
||||
files = [
|
||||
("unstructured", open(test_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ground_truth", open(ground_truth_file, "rb")),
|
||||
]
|
||||
response = requests.post(
|
||||
self._base_url + "benchmark",
|
||||
headers=headers,
|
||||
params=params,
|
||||
files=files,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif response.status_code == 400:
|
||||
raise InvalidFileException(
|
||||
response.json().get("detail", "Invalid File")
|
||||
)
|
||||
logger.debug(f"Server responded: {response.json()}")
|
||||
return response.json()["job_id"]
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_create_benchmark_job, "create benchmark request"
|
||||
)
|
||||
|
||||
def append_to_benchmark_job(
|
||||
self,
|
||||
job_id: str,
|
||||
test_file: Path,
|
||||
ontology_file: Path,
|
||||
ground_truth_file: Path,
|
||||
last_fileset=True,
|
||||
):
|
||||
"""
|
||||
Appends a new fileset to an already existing benchmark job.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: an existing job id that accepts this file set
|
||||
:param test_file: the test file path
|
||||
:param ontology_file: the ontology file path
|
||||
:param ground_truth_file: the ground truth file
|
||||
:param last_fileset: True, indicates this is the last file set, so the job can be placed in the ready queue.
|
||||
:return: the job id of the job
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
if not test_file.exists() or not test_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Input test file does not exist or is not a file: {str(test_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology file does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ground_truth_file.exists() or not ground_truth_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ground truth file does not exist or is not a file: {str(ground_truth_file)}"
|
||||
)
|
||||
|
||||
def func_append_to_benchmark_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"ready_for_processing": last_fileset}
|
||||
files = [
|
||||
("unstructured", open(test_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ground_truth", open(ground_truth_file, "rb")),
|
||||
]
|
||||
response = requests.put(
|
||||
self._base_url + f"benchmark/{job_id}",
|
||||
headers=headers,
|
||||
params=params,
|
||||
files=files,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif response.status_code == 400:
|
||||
raise InvalidFileException(
|
||||
response.json().get("detail", "Invalid File")
|
||||
)
|
||||
elif response.status_code == 404:
|
||||
raise JobNotFoundException(f"Job ID: {job_id} does not exist.")
|
||||
|
||||
logger.debug(f"Server responded: {response.json()}")
|
||||
return response.json()
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_append_to_benchmark_job, "append to benchmark request"
|
||||
)
|
||||
|
||||
def retrieve_benchmark_files(
|
||||
self, job_id: str, output_filepaths: List[Path]
|
||||
) -> List[Path]:
|
||||
"""
|
||||
Retrieves the files containing the benchmark results from the server.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the benchmark job id
|
||||
:param output_filepaths: the output filepaths where the benchmark files will be stored.
|
||||
:return: a list of file paths where the benchmark files were stored. The order of the files is the same as
|
||||
they were uploaded.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
output_filenames: List[Path] = []
|
||||
for idx in range(len(output_filepaths)):
|
||||
try:
|
||||
os.makedirs(output_filepaths[idx].parent.as_posix(), exist_ok=True)
|
||||
except OSError as error:
|
||||
error_path = str(output_filepaths[idx].parent)
|
||||
logger.log(
|
||||
logging.CRITICAL,
|
||||
f"Failed to create folder: {error_path} with cause {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise UnexpectedConditionException(
|
||||
f"Could not create output folder: {error_path}"
|
||||
)
|
||||
|
||||
def func_retrieve_benchmark_file():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"index": idx}
|
||||
response = requests.get(
|
||||
self._base_url + "benchmark/" + job_id,
|
||||
headers=headers,
|
||||
params=params,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
if response.status_code == 204:
|
||||
raise ClientException(
|
||||
f"Benchmark job {job_id} not ready at index {idx}"
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise JobNotFoundException(
|
||||
f"Benchmark output not found for job {job_id}, index {idx}"
|
||||
)
|
||||
|
||||
with open(output_filepaths[idx], "wb") as fd:
|
||||
for chunk in response.iter_content(
|
||||
chunk_size=512 * 1024
|
||||
): # 512 kBytes chunks
|
||||
fd.write(chunk)
|
||||
|
||||
return output_filepaths[idx]
|
||||
|
||||
filepath: Path = self._generic_http_request_executor(
|
||||
func_retrieve_benchmark_file, "retrieve benchmark file request"
|
||||
)
|
||||
output_filenames.append(filepath)
|
||||
|
||||
return output_filenames
|
||||
|
||||
def create_unstructured_to_kg_job(
|
||||
self,
|
||||
unstructured_text_file: Path,
|
||||
ontology_file: Path,
|
||||
ontology_spec_file: Path,
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
):
|
||||
"""
|
||||
Creates a text to KG conversion job in the server and uploads the required files.
|
||||
Login is requested if required by the server.
|
||||
If the upload is successful, it returns a job ID.
|
||||
:param unstructured_text_file: a valid file path containing unstructured text for conversion
|
||||
:param ontology_file: a valid file containing an ontology in JSON format.
|
||||
:param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format.
|
||||
:param force_filetype: indicate the file type to force for the input file
|
||||
:return: the job ID where the files were placed.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
if not unstructured_text_file.exists() or not unstructured_text_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Unstructured text file does not exist or is not a file: "
|
||||
f"{str(unstructured_text_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology files does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ontology_spec_file.exists() or not ontology_spec_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"OWL/XML ontology file does not exist or is not a file: "
|
||||
f"{str(ontology_spec_file)}"
|
||||
)
|
||||
|
||||
def func_create_unstructured_to_kg_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"file_type": force_filetype}
|
||||
files = [
|
||||
("unstructured", open(unstructured_text_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ontology_spec", open(ontology_spec_file, "rb")),
|
||||
]
|
||||
response = requests.post(
|
||||
self._base_url + "unstructured/with_ontology",
|
||||
headers=headers,
|
||||
params=params,
|
||||
files=files,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unknown reason")
|
||||
)
|
||||
elif response.status_code == 400:
|
||||
raise InvalidFileException(
|
||||
response.json().get("detail", "Unknown reason")
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
return response.json()["job_id"]
|
||||
else:
|
||||
response.raise_for_status()
|
||||
response_data = response.json()
|
||||
logger.debug(f"Server response: {response_data}")
|
||||
return response.json()["job_id"]
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_create_unstructured_to_kg_job,
|
||||
"create unstructured to KG extraction job request",
|
||||
)
|
||||
|
||||
def create_unstructured_to_kg_wildcards_job(
|
||||
self,
|
||||
unstructured_text_file: Path,
|
||||
ontology_file: Path,
|
||||
ontology_spec_file: Path,
|
||||
wildcards_query_file: Path,
|
||||
force_filetype: FileTypeAPI = FileTypeAPI.AutoDetect,
|
||||
):
|
||||
"""
|
||||
Creates a text to KG conversion job (with wildcards filtering) in the server and uploads the required files.
|
||||
Login is requested if required by the server.
|
||||
If the upload is successful, it returns a job ID.
|
||||
:param unstructured_text_file: a valid file path containing unstructured text for conversion
|
||||
:param ontology_file: a valid file containing an ontology in JSON format.
|
||||
:param ontology_spec_file: a valid file path containing the ontology spec in OWL/XML format.
|
||||
:param wildcards_query_file: a valid file path containing wildcard queries in JSON.
|
||||
:param force_filetype: indicate the file type to force for the input file
|
||||
:return: the job ID where the files were placed.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
if not unstructured_text_file.exists() or not unstructured_text_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Unstructured text file does not exist or is not a file: "
|
||||
f"{str(unstructured_text_file)}"
|
||||
)
|
||||
|
||||
if not ontology_file.exists() or not ontology_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Ontology files does not exist or is not a file: {str(ontology_file)}"
|
||||
)
|
||||
|
||||
if not ontology_spec_file.exists() or not ontology_spec_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"OWL/XML ontology file does not exist or is not a file: "
|
||||
f"{str(ontology_spec_file)}"
|
||||
)
|
||||
|
||||
if not wildcards_query_file.exists() or not wildcards_query_file.is_file():
|
||||
raise InvalidFileException(
|
||||
f"Wildcards query file does not exist or is not a file: "
|
||||
f"{str(wildcards_query_file)}"
|
||||
)
|
||||
|
||||
def func_create_unstructured_to_kg_wildcards_job():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"file_type": force_filetype}
|
||||
files = [
|
||||
("unstructured", open(unstructured_text_file, "rb")),
|
||||
("ontology", open(ontology_file, "rb")),
|
||||
("ontology_spec", open(ontology_spec_file, "rb")),
|
||||
("wildcards", open(wildcards_query_file, "rb")),
|
||||
]
|
||||
response = requests.post(
|
||||
self._base_url + "unstructured/with_wildcards",
|
||||
headers=headers,
|
||||
params=params,
|
||||
files=files,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unknown reason")
|
||||
)
|
||||
elif response.status_code == 400:
|
||||
raise InvalidFileException(
|
||||
response.json().get("detail", "Unknown reason")
|
||||
)
|
||||
|
||||
if response.status_code == 201:
|
||||
return response.json()["job_id"]
|
||||
else:
|
||||
response.raise_for_status()
|
||||
response_data = response.json()
|
||||
logger.debug(f"Server response: {response_data}")
|
||||
return response.json()["job_id"]
|
||||
|
||||
return self._generic_http_request_executor(
|
||||
func_create_unstructured_to_kg_wildcards_job,
|
||||
"create unstructured to KG extraction with wildcards job request",
|
||||
)
|
||||
|
||||
def retrieve_unstructured_to_kg_files(
|
||||
self, job_id: str, output_filepath: Path
|
||||
) -> Path:
|
||||
"""
|
||||
Retrieves the files containing the unstructured to KG results from the server.
|
||||
Login is requested if required by the server.
|
||||
:param job_id: the unstructured to kg job id
|
||||
:param output_filepath: the output filepath where the KG triplet file will be stored.
|
||||
:return: a file path where the KG file was stored.
|
||||
"""
|
||||
if self._token is None:
|
||||
self.update_token()
|
||||
|
||||
# Ensure output directory exists, but only if parent directory already exists, otherwise it can indicate a
|
||||
# configuration issue.
|
||||
try:
|
||||
os.makedirs(output_filepath.parent.as_posix(), exist_ok=True)
|
||||
except OSError as error:
|
||||
error_path = str(output_filepath.parent)
|
||||
logger.log(
|
||||
logging.CRITICAL,
|
||||
f"Failed to create folder: {error_path} with cause {error}",
|
||||
exc_info=True,
|
||||
)
|
||||
raise UnexpectedConditionException(
|
||||
f"Could not create output folder: {error_path}"
|
||||
)
|
||||
|
||||
def func_retrieve_unstructured_to_kg_file():
|
||||
headers = {"Authorization": "Bearer " + self._token}
|
||||
params = {"response_type": ResponseTypeAPI.JsonFile}
|
||||
response = requests.get(
|
||||
self._base_url + "unstructured/" + job_id,
|
||||
headers=headers,
|
||||
params=params,
|
||||
timeout=self._timeout,
|
||||
)
|
||||
# Check response status
|
||||
if response.status_code == 204:
|
||||
raise ClientException(f"Job results not ready yet for job ID: {job_id}")
|
||||
elif response.status_code == 401:
|
||||
raise UnauthorizedException(
|
||||
response.json().get("detail", "Unauthorized")
|
||||
)
|
||||
elif response.status_code == 404:
|
||||
raise JobNotFoundException(f"Job results for ID: {job_id} not found")
|
||||
|
||||
# Raise exception for any other non-success status code
|
||||
response.raise_for_status()
|
||||
|
||||
# Save the content to the output file
|
||||
with open(output_filepath, "wb") as fd:
|
||||
for chunk in response.iter_content(
|
||||
chunk_size=512 * 1024
|
||||
): # 512 kBytes chunks
|
||||
fd.write(chunk)
|
||||
|
||||
logger.info(f"Successfully wrote results to {output_filepath}")
|
||||
return output_filepath
|
||||
|
||||
filepath: Path = self._generic_http_request_executor(
|
||||
func_retrieve_unstructured_to_kg_file,
|
||||
"retrieve unstructured to KG file request",
|
||||
)
|
||||
return filepath
|
||||
@@ -0,0 +1 @@
|
||||
"""Evaluation utilities for text2kg package."""
|
||||
@@ -0,0 +1,399 @@
|
||||
"""
|
||||
This module provides functionality for detailed evaluation of entity and relation extraction.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
|
||||
class CustomEncoder(json.JSONEncoder):
|
||||
"""Custom JSON encoder to handle sets."""
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, set):
|
||||
return list(obj)
|
||||
return super().default(obj)
|
||||
|
||||
|
||||
def read_jsonl(file_path: str) -> List[Dict]:
|
||||
"""Read a JSONL file and return its contents as a list of dictionaries."""
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
|
||||
def write_jsonl(data: List[Dict], file_path: str):
|
||||
"""Write a list of dictionaries to a JSONL file."""
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
for item in data:
|
||||
json.dump(item, f, cls=CustomEncoder)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def load_ontology(ontology_path: str) -> Dict:
|
||||
"""Load an ontology from a JSON file."""
|
||||
with open(ontology_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def calculate_metrics(tp: int, fp: int, fn: int) -> Tuple[float, float, float]:
|
||||
"""Calculate precision, recall, and F1 score."""
|
||||
precision = tp / (tp + fp) if tp + fp > 0 else 0
|
||||
recall = tp / (tp + fn) if tp + fn > 0 else 0
|
||||
f1 = (
|
||||
2 * (precision * recall) / (precision + recall) if precision + recall > 0 else 0
|
||||
)
|
||||
return round(precision, 2), round(recall, 2), round(f1, 2)
|
||||
|
||||
|
||||
def normalize_string(label: str) -> str:
|
||||
"""Normalize a string by removing special characters and converting to lowercase."""
|
||||
# Remove everything in parentheses including parentheses
|
||||
label = re.sub(r"\(.*?\)", "", label)
|
||||
# Remove everything after a comma including the comma
|
||||
label = re.sub(r",.*", "", label)
|
||||
# Removes occurences of 'the' and subsequent white-spaces
|
||||
label = re.sub(r"^(The|the)\s+", "", label)
|
||||
# Remove every non-alphanumeric character and spaces
|
||||
label = re.sub(r"[^a-zA-Z0-9]", "", label)
|
||||
# Remove alphabets that appear after numbers
|
||||
label = re.sub(r"(\d+)[a-zA-Z]+", r"\1", label)
|
||||
# Remove trailing 's'
|
||||
label = re.sub(r"s$", "", label)
|
||||
# Convert to lowercase
|
||||
return label.lower()
|
||||
|
||||
|
||||
def normalize_triple(sub_label: str, rel_label: str, obj_label: str) -> str:
|
||||
"""
|
||||
Normalize triples for comparison in precision, recall calculations
|
||||
:param sub_label: subject string
|
||||
:param rel_label: relation string
|
||||
:param obj_label: object string
|
||||
:return: a normalized triple as a single concatenated string
|
||||
"""
|
||||
# remove spaces and underscores and make lower case
|
||||
if not isinstance(obj_label, str):
|
||||
obj_label = str(obj_label)
|
||||
if obj_label is None:
|
||||
obj_label = ""
|
||||
sub_label = normalize_string(sub_label)
|
||||
rel_label = normalize_string(rel_label)
|
||||
obj_label = normalize_string(obj_label)
|
||||
# concatenate them to a single string
|
||||
tr_key = f"{sub_label}{rel_label}{obj_label}"
|
||||
return tr_key
|
||||
|
||||
|
||||
def evaluate_entity_presence(ground_truth: Dict, response: Dict) -> Dict:
|
||||
"""Evaluate the presence of entities in the response compared to ground truth."""
|
||||
gt_entities = set()
|
||||
for triple in ground_truth["triples"]:
|
||||
gt_entities.add(normalize_string(triple["sub"]))
|
||||
gt_entities.add(normalize_string(triple["obj"]))
|
||||
|
||||
response_entities = set(normalize_string(ent["text"]) for ent in response["ents"])
|
||||
|
||||
tp = len(gt_entities.intersection(response_entities))
|
||||
fp = len(response_entities - gt_entities)
|
||||
fn = len(gt_entities - response_entities)
|
||||
|
||||
precision, recall, f1 = calculate_metrics(tp, fp, fn)
|
||||
|
||||
return {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"f1": f1,
|
||||
"has_entities": len(response_entities) > 0,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_entity_linking(ground_truth: Dict, response: Dict, ontology: Dict) -> Dict:
|
||||
"""Evaluate the accuracy of entity linking."""
|
||||
class_mapping = {}
|
||||
for relation, class_pairs in ontology["Relations"].items():
|
||||
for domain, range_class in class_pairs:
|
||||
for triple in ground_truth["triples"]:
|
||||
if triple["rel"] == relation:
|
||||
class_mapping[normalize_string(triple["sub"])] = domain
|
||||
class_mapping[normalize_string(triple["obj"])] = range_class
|
||||
|
||||
correct_links = 0
|
||||
total_links = 0
|
||||
gt_class_instances = []
|
||||
correctly_linked_entities = set()
|
||||
|
||||
for ent in response["ents"]:
|
||||
normalized_text = normalize_string(ent["text"])
|
||||
if normalized_text in class_mapping:
|
||||
total_links += 1
|
||||
gt_class = class_mapping[normalized_text]
|
||||
gt_class_instances.append(
|
||||
{
|
||||
"text": ent["text"],
|
||||
"gt_class": gt_class,
|
||||
"response_class": ent["class"],
|
||||
}
|
||||
)
|
||||
if ent["class"] == gt_class:
|
||||
correct_links += 1
|
||||
correctly_linked_entities.add(normalized_text)
|
||||
|
||||
accuracy = correct_links / total_links if total_links > 0 else None
|
||||
|
||||
return {
|
||||
"accuracy": round(accuracy, 2) if accuracy is not None else None,
|
||||
"correct_links": correct_links,
|
||||
"total_links": total_links,
|
||||
"gt_class_instances": gt_class_instances,
|
||||
"correctly_linked_entities": list(correctly_linked_entities),
|
||||
"has_links": total_links > 0,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_relation_linking(
|
||||
ground_truth: Dict, response: Dict, correctly_linked_entities: List[str]
|
||||
) -> Dict:
|
||||
"""Evaluate the accuracy of relation linking."""
|
||||
gt_triples = ground_truth["triples"]
|
||||
response_triples = response["triples"]
|
||||
|
||||
filtered_gt_triples = [
|
||||
triple
|
||||
for triple in gt_triples
|
||||
if normalize_string(triple["sub"]) in correctly_linked_entities
|
||||
and normalize_string(triple["obj"]) in correctly_linked_entities
|
||||
]
|
||||
|
||||
gt_dict = {
|
||||
(normalize_string(triple["sub"]), normalize_string(triple["obj"])): triple[
|
||||
"rel"
|
||||
]
|
||||
for triple in filtered_gt_triples
|
||||
}
|
||||
response_dict = {
|
||||
(normalize_string(triple[0]), normalize_string(triple[2])): triple[1]
|
||||
for triple in response_triples
|
||||
}
|
||||
|
||||
correct_triples = []
|
||||
missing_triples = []
|
||||
wrong_triples = []
|
||||
|
||||
for (sub, obj), rel in gt_dict.items():
|
||||
if (sub, obj) in response_dict:
|
||||
if response_dict[(sub, obj)] == rel:
|
||||
correct_triples.append((sub, rel, obj))
|
||||
else:
|
||||
wrong_triples.append((sub, response_dict[(sub, obj)], obj))
|
||||
else:
|
||||
missing_triples.append((sub, rel, obj))
|
||||
|
||||
total_triples = len(filtered_gt_triples)
|
||||
|
||||
return {
|
||||
"correct": (
|
||||
round(len(correct_triples) / total_triples, 2)
|
||||
if total_triples > 0
|
||||
else None
|
||||
),
|
||||
"missing": (
|
||||
round(len(missing_triples) / total_triples, 2)
|
||||
if total_triples > 0
|
||||
else None
|
||||
),
|
||||
"wrong": (
|
||||
round(len(wrong_triples) / total_triples, 2) if total_triples > 0 else None
|
||||
),
|
||||
"correct_triples": correct_triples,
|
||||
"missing_triples": missing_triples,
|
||||
"wrong_triples": wrong_triples,
|
||||
"total_triples": total_triples,
|
||||
"has_triples": total_triples > 0,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_triples(ground_truth: Dict, response: Dict) -> Dict:
|
||||
"""Evaluate the accuracy of relation linking."""
|
||||
gt_triples = ground_truth["triples"]
|
||||
response_triples = response["triples"]
|
||||
|
||||
# collect the set of relations in ground truth triples, spaces are converted to "_" to make them
|
||||
# comparable with system triples
|
||||
gt_relations = {tr["rel"].replace(" ", "_") for tr in gt_triples}
|
||||
|
||||
# filter out any triples in system output that does not match with ground truth relations
|
||||
filtered_system_triples = [tr for tr in response_triples if tr[1] in gt_relations]
|
||||
|
||||
# create a normalized string from subject, relation, object of each triple for comparison
|
||||
normalized_system_triples = {
|
||||
normalize_triple(tr[0], tr[1], tr[2]) for tr in filtered_system_triples
|
||||
}
|
||||
normalized_gt_triples = {
|
||||
normalize_triple(tr["sub"], tr["rel"], tr["obj"]) for tr in gt_triples
|
||||
}
|
||||
|
||||
# compare the system output triples with ground truth triples and calculate precision, recall, f1
|
||||
precision, recall, f1 = calculate_precision_recall_f1(
|
||||
normalized_gt_triples, normalized_system_triples
|
||||
)
|
||||
|
||||
return {"triples_precision": precision, "triples_recall": recall, "triples_f1": f1}
|
||||
|
||||
|
||||
def evaluate_ontology(ontology_id: str, config: Dict) -> Dict:
|
||||
"""Evaluate the performance for a specific ontology."""
|
||||
output_path = config["path_patterns"]["sys"].replace("$$onto$$", ontology_id)
|
||||
ground_truth_path = config["path_patterns"]["gt"].replace("$$onto$$", ontology_id)
|
||||
ontology_path = config["path_patterns"]["onto"].replace("$$onto$$", ontology_id)
|
||||
|
||||
outputs = read_jsonl(output_path)
|
||||
ground_truths = read_jsonl(ground_truth_path)
|
||||
ontology = load_ontology(ontology_path)
|
||||
|
||||
results = []
|
||||
|
||||
for gt, resp in zip(ground_truths, outputs):
|
||||
triples_evaluation = evaluate_triples(gt, resp)
|
||||
entity_presence = evaluate_entity_presence(gt, resp)
|
||||
entity_linking = evaluate_entity_linking(gt, resp, ontology)
|
||||
relation_linking = evaluate_relation_linking(
|
||||
gt, resp, entity_linking["correctly_linked_entities"]
|
||||
)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"id": gt["id"],
|
||||
"triples_evaluation": triples_evaluation,
|
||||
"entity_presence": entity_presence,
|
||||
"entity_linking": entity_linking,
|
||||
"relation_linking": relation_linking,
|
||||
"response_entities": resp["ents"],
|
||||
"response_triples": resp["triples"],
|
||||
"ground_truth_triples": gt["triples"],
|
||||
"sent": gt["sent"],
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def calculate_average_metrics(results: List[Dict], is_detailed_metrics: bool) -> Dict:
|
||||
"""Calculate average metrics across all results."""
|
||||
avg_metrics = defaultdict(lambda: {"sum": 0, "count": 0})
|
||||
for result in results:
|
||||
teval = result["triples_evaluation"]
|
||||
ep = result["entity_presence"]
|
||||
if is_detailed_metrics:
|
||||
el = result["entity_linking"]
|
||||
rl = result["relation_linking"]
|
||||
|
||||
avg_metrics["triples_evaluation_precision"]["sum"] += teval["triples_precision"]
|
||||
avg_metrics["triples_evaluation_recall"]["sum"] += teval["triples_recall"]
|
||||
avg_metrics["triples_evaluation_f1"]["sum"] += teval["triples_f1"]
|
||||
avg_metrics["triples_evaluation_precision"]["count"] += 1
|
||||
avg_metrics["triples_evaluation_recall"]["count"] += 1
|
||||
avg_metrics["triples_evaluation_f1"]["count"] += 1
|
||||
|
||||
avg_metrics["entity_presence_precision"]["sum"] += ep["precision"]
|
||||
avg_metrics["entity_presence_recall"]["sum"] += ep["recall"]
|
||||
avg_metrics["entity_presence_f1"]["sum"] += ep["f1"]
|
||||
avg_metrics["entity_presence_precision"]["count"] += 1
|
||||
avg_metrics["entity_presence_recall"]["count"] += 1
|
||||
avg_metrics["entity_presence_f1"]["count"] += 1
|
||||
|
||||
if is_detailed_metrics:
|
||||
if el["has_links"]:
|
||||
avg_metrics["entity_linking_accuracy"]["sum"] += el["accuracy"]
|
||||
avg_metrics["entity_linking_accuracy"]["count"] += 1
|
||||
|
||||
if rl["has_triples"]:
|
||||
avg_metrics["relation_linking_correct"]["sum"] += rl["correct"]
|
||||
avg_metrics["relation_linking_missing"]["sum"] += rl["missing"]
|
||||
avg_metrics["relation_linking_wrong"]["sum"] += rl["wrong"]
|
||||
avg_metrics["relation_linking_correct"]["count"] += 1
|
||||
avg_metrics["relation_linking_missing"]["count"] += 1
|
||||
avg_metrics["relation_linking_wrong"]["count"] += 1
|
||||
|
||||
return {
|
||||
key: round(value["sum"] / value["count"], 2) if value["count"] > 0 else None
|
||||
for key, value in avg_metrics.items()
|
||||
}
|
||||
|
||||
|
||||
def calculate_precision_recall_f1(gold: Set, pred: Set) -> tuple[float, float, float]:
|
||||
"""
|
||||
Method to calculate precision, recall and f1:
|
||||
Precision is calculated as correct_triples/predicted_triples and
|
||||
Recall as correct_triples/gold_triples
|
||||
F1 as the harmonic mean of precision and recall.
|
||||
:param gold: items in the gold standard
|
||||
:param pred: items in the system prediction
|
||||
:return:
|
||||
p: float - precision
|
||||
r: float - recall
|
||||
f1: float - F1
|
||||
"""
|
||||
if len(pred) == 0:
|
||||
return 0, 0, 0
|
||||
p = len(gold.intersection(pred)) / len(pred)
|
||||
r = len(gold.intersection(pred)) / len(gold)
|
||||
if p + r > 0:
|
||||
f1 = 2 * ((p * r) / (p + r))
|
||||
else:
|
||||
f1 = 0
|
||||
return p, r, f1
|
||||
|
||||
|
||||
def EvaluateResults(config: Dict, is_detailed_metrics: bool = True):
|
||||
ontology_ids = config["onto_list"]
|
||||
overall_avg_metrics = defaultdict(list)
|
||||
|
||||
for ontology_id in ontology_ids:
|
||||
results = evaluate_ontology(ontology_id, config)
|
||||
new_results = results
|
||||
if not is_detailed_metrics:
|
||||
new_results = []
|
||||
for result in results:
|
||||
new_results.append(
|
||||
{
|
||||
"id": result["id"],
|
||||
"entity_presence": result["entity_presence"],
|
||||
"response_entities": result["response_entities"],
|
||||
"sent": result["sent"],
|
||||
}
|
||||
)
|
||||
# all_ontology_results[ontology_id] = new_results
|
||||
|
||||
# Write detailed results for each ontology
|
||||
output_path = config["path_patterns"]["output"].replace("$$onto$$", ontology_id)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
write_jsonl(new_results, output_path)
|
||||
|
||||
# Calculate average metrics for each ontology
|
||||
avg_metrics = calculate_average_metrics(results, is_detailed_metrics)
|
||||
for key, value in avg_metrics.items():
|
||||
if value is not None:
|
||||
overall_avg_metrics[key].append((ontology_id, value))
|
||||
|
||||
# Calculate overall average metrics and write all results to a single file
|
||||
overall_avg_output_path = config["overall_output"]
|
||||
os.makedirs(os.path.dirname(overall_avg_output_path), exist_ok=True)
|
||||
with open(overall_avg_output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"per_ontology_metrics": {
|
||||
key: {ont_id: value for ont_id, value in values}
|
||||
for key, values in overall_avg_metrics.items()
|
||||
},
|
||||
"global_avg_metrics": {
|
||||
key: round(sum(v for _, v in values) / len(values), 2)
|
||||
for key, values in overall_avg_metrics.items()
|
||||
},
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
cls=CustomEncoder,
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
Module that contains Exceptions that maybe thrown by the Benchmark client.
|
||||
|
||||
Copyright (c) 2016 - present Syncleus, Inc.
|
||||
Copyright (c) 2016 - present CleverThis, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
|
||||
class ClientException(Exception):
|
||||
"""
|
||||
Client exception occurred.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class BaseUrlMissingException(ClientException):
|
||||
"""
|
||||
Base URL is missing. No REST operations are possible.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class InvalidUsernameOrPasswordException(ClientException):
|
||||
"""
|
||||
Invalid username or password.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class UnexpectedConditionException(ClientException):
|
||||
"""
|
||||
Something unexpected happened during processing. Some unhandled case that should not occur.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RequestErrorException(ClientException):
|
||||
"""
|
||||
Request is badly formatted or server error occurred.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ConnectionErrorException(ClientException):
|
||||
"""
|
||||
Failed to connect to server.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ConnectionTimeoutException(ClientException):
|
||||
"""
|
||||
Connection to server timed out.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class JobNotFoundException(ClientException):
|
||||
"""
|
||||
Job not found.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class UnauthorizedException(ClientException):
|
||||
"""
|
||||
Unauthorized exception. Either user is not logged in or is not authorized to do the operation.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class InvalidFileException(ClientException):
|
||||
"""
|
||||
File is not valid.
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,14 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class FileTypeAPI(StrEnum):
|
||||
AutoDetect = "AutoDetect"
|
||||
Markdown = "Markdown"
|
||||
PlainText = "PlainText"
|
||||
|
||||
|
||||
class FileType(StrEnum):
|
||||
JSONL = "JSONL"
|
||||
Markdown = "Markdown"
|
||||
PlainText = "PlainText"
|
||||
Unknown = "Unknown"
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
Module that contains the enumerations found when interfacing with Benchmark client and server.
|
||||
|
||||
Copyright (c) 2016 - present Syncleus, Inc.
|
||||
Copyright (c) 2016 - present CleverThis, Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class JobType(StrEnum):
|
||||
UnstructuredWithOntology = "UnstructuredWithOntology"
|
||||
UnstructuredWithWildcards = "UnstructuredWithWildcards"
|
||||
Benchmark = "Benchmark"
|
||||
|
||||
|
||||
class JobStatus(StrEnum):
|
||||
Created = "Created"
|
||||
ReadyForProcessing = "ReadyForProcessing"
|
||||
Processing = "Processing"
|
||||
Completed = "Completed"
|
||||
Failed = "Failed"
|
||||
@@ -0,0 +1,6 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ResponseTypeAPI(StrEnum):
|
||||
JsonText = "JsonText"
|
||||
JsonFile = "JsonFile"
|
||||
Reference in New Issue
Block a user