feat/#1 - refactor code to allow make it as library and to separate the AMQ code from dependencies needed to write the CleverSwarm interface
This commit is contained in:
+42
-150
@@ -11,11 +11,8 @@ from amqp.adapter.amq_response_factory import AMQResponseFactory
|
|||||||
from amqp.config.amq_configuration import AMQConfiguration
|
from amqp.config.amq_configuration import AMQConfiguration
|
||||||
from amqp.model.model import AMQMessage, AMQResponse
|
from amqp.model.model import AMQMessage, AMQResponse
|
||||||
|
|
||||||
import rest.v0.endpoints.benchmark as benchmark
|
from cleverswarm.core.deps import get_current_user
|
||||||
import rest.v0.endpoints.jobs as jobs
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
import rest.v0.endpoints.unstructured as unstructured
|
|
||||||
from core.deps import get_current_user
|
|
||||||
from schemas.user_schema import UserSchema
|
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from urllib.parse import parse_qs
|
from urllib.parse import parse_qs
|
||||||
@@ -94,17 +91,6 @@ def parse_request_body(message: AMQMessage):
|
|||||||
else:
|
else:
|
||||||
raise ValueError(f"Unsupported Content-Type: {content_type}")
|
raise ValueError(f"Unsupported Content-Type: {content_type}")
|
||||||
|
|
||||||
|
|
||||||
BENCHMARK = '/benchmark'
|
|
||||||
JOBS = '/jobs/'
|
|
||||||
UNSTRUCTURED = '/unstructured'
|
|
||||||
|
|
||||||
GET_BENCHMARK_KG = r".*/benchmark/(?P<request_id>[^\?]+)\?index=(?P<index>\d+)"
|
|
||||||
GET_JOB_STATUS = r".*/jobs/(?P<request_id>.*)"
|
|
||||||
GET_KG = r".*/unstructured/(?P<request_id>.*)"
|
|
||||||
DUMMY = r"(?P<request_id>.*)"
|
|
||||||
|
|
||||||
|
|
||||||
# ============= POST Helper function =============
|
# ============= POST Helper function =============
|
||||||
|
|
||||||
async def call_cleverswarm_post_put_api(
|
async def call_cleverswarm_post_put_api(
|
||||||
@@ -128,33 +114,6 @@ async def call_cleverswarm_post_put_api(
|
|||||||
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
|
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
|
||||||
|
|
||||||
|
|
||||||
#
|
|
||||||
# Following methods are wrappers to actual CleverSwarm endpoints, so I can refer to the CleverSwarm method and
|
|
||||||
# pass it as input parameter using general Callable[[Any, UserSchema], str] type
|
|
||||||
#
|
|
||||||
async def cleverswarm_create_benchmark_job(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await benchmark.create_benchmark_job(**args, authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_create_extract_with_ontology(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await unstructured.create_extract_with_ontology(**args, authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_create_extract_with_wildcards(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await unstructured.create_extract_with_ontology(**args, authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_update_benchmark_job(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await benchmark.update_benchmark_job(**args, authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_retry_job(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await jobs.retry_job(args[0], authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_delete_job(args: Any, authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await jobs.delete_job(args[0], authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
# ============= GET Helper function =============
|
# ============= GET Helper function =============
|
||||||
async def call_cleverswarm_get_api(
|
async def call_cleverswarm_get_api(
|
||||||
@@ -179,31 +138,32 @@ async def call_cleverswarm_get_api(
|
|||||||
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
|
return AMQResponseFactory.of(message.id, 500, "text/plain", str(error).encode('utf-8'), message.trace_info)
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_get_benchmark_kg(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
# ============= CleverThisServiceAdapter =============
|
||||||
return str(await benchmark.get_benchmark_kg(args[0], args[1], authenticated_user=authenticated_user))
|
class CleverThisServiceAdapter:
|
||||||
|
"""
|
||||||
|
A base class for CleverThis service adapter that handles the incoming AMQP messages and routes them to the
|
||||||
async def cleverswarm_get_job_status(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
appropriate CleverThisService API endpoint.
|
||||||
return str(await jobs.get_job_status(args[0], authenticated_user=authenticated_user))
|
"""
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_jobs_get_requests(args: Tuple[AnyStr | Any, ...],
|
|
||||||
authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await jobs.get_requests(authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
async def cleverswarm_get_kg(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
|
||||||
return str(await unstructured.get_kg(args[0], authenticated_user=authenticated_user))
|
|
||||||
|
|
||||||
|
|
||||||
class CleverSwarmAdapter:
|
|
||||||
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
|
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
|
||||||
self.amq_configuration = amq_configuration
|
self.amq_configuration = amq_configuration
|
||||||
self.session = session
|
self.session = session
|
||||||
self.service_host = self.amq_configuration.dispatch.service_host
|
self.service_host = self.amq_configuration.dispatch.service_host
|
||||||
self.service_port = self.amq_configuration.dispatch.service_port
|
self.service_port = self.amq_configuration.dispatch.service_port
|
||||||
|
|
||||||
|
async def get_current_user(self, token: str):
|
||||||
|
"""
|
||||||
|
To Be Overriden in subclass for given CleverThis Service, to return the current user based on the token,
|
||||||
|
in format appropriate for the service.
|
||||||
|
:return: A user object or None if the user is not authenticated.
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
async def on_message(self, message) -> AMQResponse:
|
async def on_message(self, message) -> AMQResponse:
|
||||||
|
"""
|
||||||
|
Called when a message is received from the AMQP.
|
||||||
|
:param message:
|
||||||
|
:return: AMQP response class with operation status code and optional error details.
|
||||||
|
"""
|
||||||
DEBUG_NO_AUTH = 1
|
DEBUG_NO_AUTH = 1
|
||||||
_auth: str = message.headers.get('Authorization', '')
|
_auth: str = message.headers.get('Authorization', '')
|
||||||
_auth_user: UserSchema | None = None
|
_auth_user: UserSchema | None = None
|
||||||
@@ -231,47 +191,17 @@ class CleverSwarmAdapter:
|
|||||||
return AMQResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
|
return AMQResponseFactory.of(message.id, 401, "text/plain", b"Unauthorized", message.trace_info)
|
||||||
|
|
||||||
# ==================================================================================================
|
# ==================================================================================================
|
||||||
# ===================== C l e v e r S w a r m A P I m e t h o d s ============================
|
# ===================== C l e v e r T h i s A P I m e t h o d s ============================
|
||||||
# ==================================================================================================
|
# ==================================================================================================
|
||||||
|
|
||||||
async def get(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
async def get(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
"""
|
"""
|
||||||
Handles the GET requests for the CleverSwarm API.
|
Handles the GET requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
|
||||||
|
It will attempt to call the REST endpoint directly using HTTP protocol.
|
||||||
:param message: message from the AMQP that contains the GET request
|
:param message: message from the AMQP that contains the GET request
|
||||||
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
"""
|
"""
|
||||||
if BENCHMARK in message.path:
|
|
||||||
return await call_cleverswarm_get_api(
|
|
||||||
message,
|
|
||||||
GET_BENCHMARK_KG,
|
|
||||||
cleverswarm_get_benchmark_kg,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
if JOBS in message.path:
|
|
||||||
return await call_cleverswarm_get_api(
|
|
||||||
message,
|
|
||||||
GET_JOB_STATUS,
|
|
||||||
cleverswarm_get_job_status,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
if message.path.endswith('/jobs'):
|
|
||||||
return await call_cleverswarm_get_api(
|
|
||||||
message,
|
|
||||||
DUMMY,
|
|
||||||
cleverswarm_jobs_get_requests,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
if UNSTRUCTURED in message.path:
|
|
||||||
return await call_cleverswarm_get_api(
|
|
||||||
message,
|
|
||||||
GET_KG,
|
|
||||||
cleverswarm_get_kg,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
# unmatched path - try to invoke the service directly on this path
|
# unmatched path - try to invoke the service directly on this path
|
||||||
url = f"http://{self.service_host}:{self.service_port}{message.path}"
|
url = f"http://{self.service_host}:{self.service_port}{message.path}"
|
||||||
@@ -287,29 +217,12 @@ class CleverSwarmAdapter:
|
|||||||
|
|
||||||
async def put(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
async def put(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
"""
|
"""
|
||||||
Handles the PUT requests for the CleverSwarm API.
|
Handles the PUT requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
|
||||||
|
It will attempt to call the REST endpoint directly using HTTP protocol.
|
||||||
:param message: message from the AMQP that contains the PUT request
|
:param message: message from the AMQP that contains the PUT request
|
||||||
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
"""
|
"""
|
||||||
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
|
||||||
if BENCHMARK in message.path:
|
|
||||||
return await call_cleverswarm_post_put_api(
|
|
||||||
message,
|
|
||||||
parser.form_data,
|
|
||||||
cleverswarm_create_benchmark_job,
|
|
||||||
201,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
if JOBS in message.path:
|
|
||||||
return await call_cleverswarm_get_api(
|
|
||||||
message,
|
|
||||||
GET_JOB_STATUS,
|
|
||||||
cleverswarm_retry_job,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
return await self.handle_possible_form(
|
return await self.handle_possible_form(
|
||||||
self.session.put(
|
self.session.put(
|
||||||
f"http://{self.service_host}:{self.service_port}{message.path}",
|
f"http://{self.service_host}:{self.service_port}{message.path}",
|
||||||
@@ -321,38 +234,12 @@ class CleverSwarmAdapter:
|
|||||||
|
|
||||||
async def post(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
async def post(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
"""
|
"""
|
||||||
Handles the POST requests for the CleverSwarm API.
|
Handles the POST requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
|
||||||
|
It will attempt to call the REST endpoint directly using HTTP protocol.
|
||||||
:param message: message from the AMQP that contains the POST request, including body and headers
|
:param message: message from the AMQP that contains the POST request, including body and headers
|
||||||
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
"""
|
"""
|
||||||
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
|
||||||
if BENCHMARK in message.path:
|
|
||||||
return await call_cleverswarm_post_put_api(
|
|
||||||
message,
|
|
||||||
parser.form_data,
|
|
||||||
cleverswarm_create_benchmark_job,
|
|
||||||
201,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
if UNSTRUCTURED in message.path:
|
|
||||||
if 'with_ontology' in message.path:
|
|
||||||
return await call_cleverswarm_post_put_api(
|
|
||||||
message,
|
|
||||||
parser.form_data,
|
|
||||||
cleverswarm_create_extract_with_ontology,
|
|
||||||
200,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
if 'with_wildcards' in message.path:
|
|
||||||
return await call_cleverswarm_post_put_api(
|
|
||||||
message,
|
|
||||||
parser.form_data,
|
|
||||||
cleverswarm_create_extract_with_wildcards,
|
|
||||||
200,
|
|
||||||
authenticated_user=auth_user
|
|
||||||
)
|
|
||||||
|
|
||||||
return await self.handle_possible_form(
|
return await self.handle_possible_form(
|
||||||
self.session.post(
|
self.session.post(
|
||||||
f"http://{self.service_host}:{self.service_port}{message.path}",
|
f"http://{self.service_host}:{self.service_port}{message.path}",
|
||||||
@@ -366,18 +253,23 @@ class CleverSwarmAdapter:
|
|||||||
return await request_coroutine
|
return await request_coroutine
|
||||||
|
|
||||||
async def head(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
async def head(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
"""
|
||||||
|
Handles the HEAD requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
|
||||||
|
It will attempt to call the REST endpoint directly using HTTP protocol.
|
||||||
|
:param message: message from the AMQP that contains the POST request, including body and headers
|
||||||
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
async def delete(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
async def delete(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
"""
|
||||||
if JOBS in message.path:
|
Handles the DELETE requests for the CleverThis API. to be overriden in subclass. This is a base implementation.
|
||||||
return await call_cleverswarm_get_api(
|
It will attempt to call the REST endpoint directly using HTTP protocol.
|
||||||
message,
|
:param message: message from the AMQP that contains the POST request, including body and headers
|
||||||
GET_JOB_STATUS,
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
cleverswarm_retry_job,
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
authenticated_user=auth_user
|
"""
|
||||||
)
|
|
||||||
|
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
|
def get_content_type(self, message_headers: Dict[str, List[str]]) -> str:
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
import asyncio
|
|
||||||
import logging
|
import logging
|
||||||
import logging.config
|
import logging.config
|
||||||
import os
|
import os
|
||||||
@@ -7,11 +6,11 @@ from asyncio import Future
|
|||||||
|
|
||||||
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
||||||
from amqp.adapter.amq_route_factory import AMQRouteFactory
|
from amqp.adapter.amq_route_factory import AMQRouteFactory
|
||||||
from amqp.adapter.clever_swarm_adapter import CleverSwarmAdapter
|
from cleverswarm.clever_swarm_adapter import CleverSwarmAdapter
|
||||||
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
|
from amqp.adapter.service_message_factory import CleverMicroServiceMessageFactory
|
||||||
|
|
||||||
from amqp.config.amq_configuration import AMQConfiguration
|
from amqp.config.amq_configuration import AMQConfiguration
|
||||||
from amqp.model.model import AMQMessage, AMQRoute
|
from amqp.model.model import AMQMessage
|
||||||
from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
|
from amqp.rabbitmq.rabbit_mq_consumer import RabbitMQConsumer
|
||||||
from amqp.service.service_message_handler import AMQServiceMessageHandler
|
from amqp.service.service_message_handler import AMQServiceMessageHandler
|
||||||
from amqp.service.tracing import initialize_trace
|
from amqp.service.tracing import initialize_trace
|
||||||
@@ -91,4 +90,4 @@ if __name__ == "__main__":
|
|||||||
amq_service: AMQService = AMQService(amq_configuration, CleverSwarmAdapter(amq_configuration))
|
amq_service: AMQService = AMQService(amq_configuration, CleverSwarmAdapter(amq_configuration))
|
||||||
amq_service.rabbit_mq_consumer.channel.start_consuming()
|
amq_service.rabbit_mq_consumer.channel.start_consuming()
|
||||||
# amq_service.consumer_thread.cancel()
|
# amq_service.consumer_thread.cancel()
|
||||||
# amq_service.rabbit_mq_consumer.connection.close()
|
# amq_service.rabbit_mq_consumer.connection.close()
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from pika.channel import Channel
|
|||||||
|
|
||||||
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
from amqp.adapter.amq_message_factory import AMQMessageFactory
|
||||||
from amqp.adapter.amq_response_factory import AMQResponseFactory
|
from amqp.adapter.amq_response_factory import AMQResponseFactory
|
||||||
from amqp.adapter.clever_swarm_adapter import CleverSwarmAdapter
|
from cleverswarm.clever_swarm_adapter import CleverSwarmAdapter
|
||||||
from amqp.model.model import AMQResponse, AMQMessage
|
from amqp.model.model import AMQResponse, AMQMessage
|
||||||
from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
|
from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
|
||||||
from amqp.router.serializer import map_as_string
|
from amqp.router.serializer import map_as_string
|
||||||
|
|||||||
@@ -0,0 +1,226 @@
|
|||||||
|
import logging
|
||||||
|
from typing import Tuple, AnyStr, Any
|
||||||
|
from aiohttp import ClientSession, ClientResponse
|
||||||
|
|
||||||
|
import cleverswarm.core.deps
|
||||||
|
from amqp.adapter.amq_response_factory import AMQResponseFactory
|
||||||
|
from amqp.adapter.cleverthis_service_adapter import CleverThisServiceAdapter, CleverMultiPartParser, \
|
||||||
|
call_cleverswarm_get_api, call_cleverswarm_post_put_api
|
||||||
|
from amqp.config.amq_configuration import AMQConfiguration
|
||||||
|
from amqp.model.model import AMQMessage, AMQResponse
|
||||||
|
|
||||||
|
import cleverswarm.rest.v0.endpoints.benchmark as benchmark
|
||||||
|
import cleverswarm.rest.v0.endpoints.jobs as jobs
|
||||||
|
import cleverswarm.rest.v0.endpoints.unstructured as unstructured
|
||||||
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
|
|
||||||
|
|
||||||
|
BENCHMARK = '/benchmark'
|
||||||
|
JOBS = '/jobs/'
|
||||||
|
UNSTRUCTURED = '/unstructured'
|
||||||
|
|
||||||
|
GET_BENCHMARK_KG = r".*/benchmark/(?P<request_id>[^\?]+)\?index=(?P<index>\d+)"
|
||||||
|
GET_JOB_STATUS = r".*/jobs/(?P<request_id>.*)"
|
||||||
|
GET_KG = r".*/unstructured/(?P<request_id>.*)"
|
||||||
|
DUMMY = r"(?P<request_id>.*)"
|
||||||
|
|
||||||
|
#
|
||||||
|
# Following methods are wrappers to actual CleverSwarm endpoints, so I can refer to the CleverSwarm method and
|
||||||
|
# pass it as input parameter using general Callable[[Any, UserSchema], str] type
|
||||||
|
#
|
||||||
|
async def cleverswarm_create_benchmark_job(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await benchmark.create_benchmark_job(**args, authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_create_extract_with_ontology(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await unstructured.create_extract_with_ontology(**args, authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_create_extract_with_wildcards(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await unstructured.create_extract_with_ontology(**args, authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_update_benchmark_job(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await benchmark.update_benchmark_job(**args, authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_retry_job(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await jobs.retry_job(args[0], authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_delete_job(args: Any, authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await jobs.delete_job(args[0], authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_get_benchmark_kg(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await benchmark.get_benchmark_kg(args[0], args[1], authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_get_job_status(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await jobs.get_job_status(args[0], authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_jobs_get_requests(args: Tuple[AnyStr | Any, ...],
|
||||||
|
authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await jobs.get_requests(authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
async def cleverswarm_get_kg(args: Tuple[AnyStr | Any, ...], authenticated_user: UserSchema) -> str:
|
||||||
|
return str(await unstructured.get_kg(args[0], authenticated_user=authenticated_user))
|
||||||
|
|
||||||
|
|
||||||
|
class CleverSwarmAdapter(CleverThisServiceAdapter):
|
||||||
|
def __init__(self, amq_configuration: AMQConfiguration, session: ClientSession = None):
|
||||||
|
super().__init__(amq_configuration, session)
|
||||||
|
|
||||||
|
async def get_current_user(self, token: str) -> UserSchema:
|
||||||
|
return await cleverswarm.core.deps.get_current_user(token)
|
||||||
|
# ==================================================================================================
|
||||||
|
# ===================== C l e v e r S w a r m A P I m e t h o d s ============================
|
||||||
|
# ==================================================================================================
|
||||||
|
|
||||||
|
async def get(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
"""
|
||||||
|
Handles the GET requests for the CleverSwarm API.
|
||||||
|
:param message: message from the AMQP that contains the GET request
|
||||||
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
|
"""
|
||||||
|
if BENCHMARK in message.path:
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
GET_BENCHMARK_KG,
|
||||||
|
cleverswarm_get_benchmark_kg,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
if JOBS in message.path:
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
GET_JOB_STATUS,
|
||||||
|
cleverswarm_get_job_status,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
if message.path.endswith('/jobs'):
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
DUMMY,
|
||||||
|
cleverswarm_jobs_get_requests,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
if UNSTRUCTURED in message.path:
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
GET_KG,
|
||||||
|
cleverswarm_get_kg,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
# unmatched path - try to invoke the service directly on this path
|
||||||
|
url = f"http://{self.service_host}:{self.service_port}{message.path}"
|
||||||
|
headers = {**message.headers, **message.trace_info}
|
||||||
|
logging.info(f" [*] SEND HEADERS out: {', '.join(f'{k}={v}' for k, v in headers.items())}, REQ={url}")
|
||||||
|
if self.session:
|
||||||
|
_http_resp: ClientResponse = await self.session.get(url, headers=headers)
|
||||||
|
return AMQResponseFactory.of(message.id, _http_resp.status, _http_resp.content_type,
|
||||||
|
await _http_resp.read(),
|
||||||
|
message.trace_info)
|
||||||
|
return AMQResponseFactory.of(message.id, 404, "text/plain",
|
||||||
|
str("Destination location does not exists").encode('utf-8'), message.trace_info)
|
||||||
|
|
||||||
|
async def put(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
"""
|
||||||
|
Handles the PUT requests for the CleverSwarm API.
|
||||||
|
:param message: message from the AMQP that contains the PUT request
|
||||||
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
|
"""
|
||||||
|
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
||||||
|
if BENCHMARK in message.path:
|
||||||
|
return await call_cleverswarm_post_put_api(
|
||||||
|
message,
|
||||||
|
parser.form_data,
|
||||||
|
cleverswarm_create_benchmark_job,
|
||||||
|
201,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
if JOBS in message.path:
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
GET_JOB_STATUS,
|
||||||
|
cleverswarm_retry_job,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
return await self.handle_possible_form(
|
||||||
|
self.session.put(
|
||||||
|
f"http://{self.service_host}:{self.service_port}{message.path}",
|
||||||
|
data=message.body,
|
||||||
|
headers={**message.headers, **message.trace_info},
|
||||||
|
),
|
||||||
|
auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
async def post(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
"""
|
||||||
|
Handles the POST requests for the CleverSwarm API.
|
||||||
|
:param message: message from the AMQP that contains the POST request, including body and headers
|
||||||
|
:param auth_user: user authenticated by the API Gateway, as UserSchem
|
||||||
|
:return: AMQResponse to be sent back to the API Gateway via AMQP
|
||||||
|
"""
|
||||||
|
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
||||||
|
if BENCHMARK in message.path:
|
||||||
|
return await call_cleverswarm_post_put_api(
|
||||||
|
message,
|
||||||
|
parser.form_data,
|
||||||
|
cleverswarm_create_benchmark_job,
|
||||||
|
201,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
if UNSTRUCTURED in message.path:
|
||||||
|
if 'with_ontology' in message.path:
|
||||||
|
return await call_cleverswarm_post_put_api(
|
||||||
|
message,
|
||||||
|
parser.form_data,
|
||||||
|
cleverswarm_create_extract_with_ontology,
|
||||||
|
200,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
if 'with_wildcards' in message.path:
|
||||||
|
return await call_cleverswarm_post_put_api(
|
||||||
|
message,
|
||||||
|
parser.form_data,
|
||||||
|
cleverswarm_create_extract_with_wildcards,
|
||||||
|
200,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
return await self.handle_possible_form(
|
||||||
|
self.session.post(
|
||||||
|
f"http://{self.service_host}:{self.service_port}{message.path}",
|
||||||
|
data=message.body,
|
||||||
|
headers={**message.headers, **message.trace_info},
|
||||||
|
),
|
||||||
|
auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_possible_form(self, request_coroutine, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
return await request_coroutine
|
||||||
|
|
||||||
|
async def head(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
async def delete(self, message: AMQMessage, auth_user: UserSchema) -> AMQResponse:
|
||||||
|
parser: CleverMultiPartParser = CleverMultiPartParser(message)
|
||||||
|
if JOBS in message.path:
|
||||||
|
return await call_cleverswarm_get_api(
|
||||||
|
message,
|
||||||
|
GET_JOB_STATUS,
|
||||||
|
cleverswarm_retry_job,
|
||||||
|
authenticated_user=auth_user
|
||||||
|
)
|
||||||
|
|
||||||
|
raise NotImplementedError
|
||||||
@@ -7,9 +7,9 @@ from fastapi.security import OAuth2PasswordBearer
|
|||||||
|
|
||||||
import jwt
|
import jwt
|
||||||
|
|
||||||
from core.configs import settings
|
from cleverswarm.core.configs import settings
|
||||||
from core.security import validate_pass, create_password_hash
|
from cleverswarm.core.security import validate_pass, create_password_hash
|
||||||
from schemas.user_schema import UserSchema
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
|
|
||||||
oauth2_schema = OAuth2PasswordBearer(
|
oauth2_schema = OAuth2PasswordBearer(
|
||||||
tokenUrl=f"{settings.API_V0_STR}/login"
|
tokenUrl=f"{settings.API_V0_STR}/login"
|
||||||
@@ -19,7 +19,7 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
JOBS_FILE_PATH: Path = 'jobs'
|
JOBS_FILE_PATH: Path = 'jobs'
|
||||||
|
|
||||||
BASE_PATH: Path = '.'
|
BASE_PATH: Path = ''
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
case_sensitive = True
|
case_sensitive = True
|
||||||
@@ -7,10 +7,9 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
|||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from schemas.user_schema import UserSchema
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
from core.auth import oauth2_schema, authenticate
|
from cleverswarm.core.auth import oauth2_schema, authenticate
|
||||||
#from core.configs import settings
|
from cleverswarm.core.security import create_password_hash
|
||||||
from core.security import create_password_hash
|
|
||||||
|
|
||||||
class TokenData(BaseModel):
|
class TokenData(BaseModel):
|
||||||
userid: Optional[str] = None
|
userid: Optional[str] = None
|
||||||
@@ -1,16 +1,16 @@
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import APIRouter, status, Depends, HTTPException
|
from fastapi import APIRouter, status, Depends
|
||||||
from fastapi import Path as FPath
|
from fastapi import Path as FPath
|
||||||
from fastapi import Query
|
from fastapi import Query
|
||||||
from fastapi import UploadFile
|
from fastapi import UploadFile
|
||||||
from fastapi.responses import FileResponse, Response
|
from fastapi.responses import Response
|
||||||
|
|
||||||
from schemas.request_schema import RequestSchema
|
from cleverswarm.schemas.request_schema import RequestSchema
|
||||||
from schemas.user_schema import UserSchema
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
from schemas.job_id_schema import JobIdSchema
|
from cleverswarm.schemas.job_id_schema import JobIdSchema
|
||||||
|
|
||||||
from core.deps import get_current_user
|
from cleverswarm.core.deps import get_current_user
|
||||||
|
|
||||||
logger = logging.getLogger('Benchmark')
|
logger = logging.getLogger('Benchmark')
|
||||||
# Enable propagation to the root logger
|
# Enable propagation to the root logger
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
from typing import List, Optional, Annotated
|
from typing import List
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import APIRouter, status, Depends, HTTPException
|
from fastapi import APIRouter, status, Depends
|
||||||
from fastapi import Path as FPath
|
from fastapi import Path as FPath
|
||||||
|
|
||||||
from schemas.request_schema import RequestSchema
|
from cleverswarm.schemas.request_schema import RequestSchema
|
||||||
from schemas.user_schema import UserSchema
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
|
|
||||||
from core.deps import get_current_user
|
from cleverswarm.core.deps import get_current_user
|
||||||
|
|
||||||
logger = logging.getLogger('Jobs')
|
logger = logging.getLogger('Jobs')
|
||||||
# Enable propagation to the root logger
|
# Enable propagation to the root logger
|
||||||
@@ -71,4 +71,4 @@ async def retry_job(request_id: str = FPath(title='Job ID',
|
|||||||
description='The id of the job to be retried'),
|
description='The id of the job to be retried'),
|
||||||
authenticated_user: UserSchema = Depends(get_current_user)):
|
authenticated_user: UserSchema = Depends(get_current_user)):
|
||||||
logging.info(f'===> PUT /jobs/{request_id}: Retries execution of a given job')
|
logging.info(f'===> PUT /jobs/{request_id}: Retries execution of a given job')
|
||||||
return RequestSchema(job_id=request_id, status='Failed')
|
return RequestSchema(job_id=request_id, status='Failed')
|
||||||
@@ -4,10 +4,10 @@ from fastapi import APIRouter, status, Depends
|
|||||||
from fastapi import Path as FPath
|
from fastapi import Path as FPath
|
||||||
from fastapi import UploadFile
|
from fastapi import UploadFile
|
||||||
|
|
||||||
from schemas.user_schema import UserSchema
|
from cleverswarm.schemas.user_schema import UserSchema
|
||||||
from schemas.job_id_schema import JobIdSchema
|
from cleverswarm.schemas.job_id_schema import JobIdSchema
|
||||||
|
|
||||||
from core.deps import get_current_user
|
from cleverswarm.core.deps import get_current_user
|
||||||
|
|
||||||
logger = logging.getLogger('Unstructured')
|
logger = logging.getLogger('Unstructured')
|
||||||
# Enable propagation to the root logger
|
# Enable propagation to the root logger
|
||||||
Reference in New Issue
Block a user