diff --git a/amqp/adapter/cleverthis_service_adapter.py b/amqp/adapter/cleverthis_service_adapter.py index f09e843..3e4c66b 100644 --- a/amqp/adapter/cleverthis_service_adapter.py +++ b/amqp/adapter/cleverthis_service_adapter.py @@ -4,6 +4,7 @@ Methods here will invoke the actual CleverThis Service code, which are passed as """ import asyncio +import enum import inspect import json import pathlib @@ -19,6 +20,7 @@ from aio_pika.abc import ( ) from aiohttp import ClientResponse, ClientSession from fastapi import HTTPException, UploadFile +from fastapi.params import File from fastapi.routing import APIRoute from fastapi.security import OAuth2PasswordRequestForm from pydantic import BaseModel @@ -29,12 +31,21 @@ from amqp.adapter import serializer from amqp.adapter.data_parser import AMQDataParser from amqp.adapter.data_response_factory import DataResponseFactory from amqp.adapter.file_handler import FileHandler, StreamingFileHandler -from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info +from amqp.adapter.logging_utils import ( + logging_debug, + logging_error, + logging_info, + logging_warning, +) +from amqp.adapter.pydantic_serializer import python_type_to_json +from amqp.adapter.type_descriptor import TypeDescriptor from amqp.config.amq_configuration import AMQConfiguration from amqp.model.model import AMQErrorMessage, AMQMessage, DataMessage, DataResponse from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE from amqp.router.utils import await_result +g_extra_init_data: dict = dict() + async def _create_reply_exchange_and_queue( connection: AbstractRobustConnection, @@ -100,6 +111,8 @@ async def _convert_file_response( def is_likely_json(text: str) -> bool: """Quick test to see if should convert the response string to JSON format.""" + if not isinstance(text, str): + return False _text = text.strip() return (_text.startswith("{") and _text.endswith("}")) or ( _text.startswith("[") and _text.endswith("]") @@ -120,12 +133,17 @@ class CleverThisServiceAdapter: amq_configuration: AMQConfiguration, routes: List[APIRoute], session: ClientSession = None, + authenticated_user_arg_name: str = "authenticated_user", + extra_init_data=None, ): """ Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP session. The HTTP session is for the case when the adapter uses loose coupling with the CleverThis service via HTTP REST API. """ + if extra_init_data is None: + extra_init_data = dict() + self.authenticated_user_arg_name = authenticated_user_arg_name self.amq_configuration = amq_configuration self.session = session self.service_host = self.amq_configuration.dispatch.service_host @@ -135,6 +153,8 @@ class CleverThisServiceAdapter: self.amq_configuration.amq_adapter.require_authenticated_user ) self.routes = self.group_routes_by_method(api_routes=routes) + global g_extra_init_data + g_extra_init_data = extra_init_data or {} def get_endpoint(self, endpoint: Any) -> Any: """ @@ -474,14 +494,15 @@ class CleverThisServiceAdapter: _match = _route.path_regex.match(_path) if _match: _args.update(_match.groupdict()) - _args["authenticated_user"] = auth_user + _signature_args = inspect.signature(_route.endpoint).parameters + if self.authenticated_user_arg_name in _signature_args.keys(): + _args[self.authenticated_user_arg_name] = auth_user # If the path matches, call the corresponding function - return await CleverThisServiceAdapter.call_cleverthis_api( + return await self.call_cleverthis_api( message, _args, api_method=_route.endpoint, success_code=_route.status_code, - loop=self.loop, ) return DataResponseFactory.of( @@ -513,40 +534,45 @@ class CleverThisServiceAdapter: _form_data.update(_query_args) for _body_param in _route.dependant.body_params: if _body_param.name in _form_data: - _file_data = _form_data[_body_param.name] - actual_filepath = message.extra_data[_body_param.name] + _file_data = _form_data.get(_body_param.name, None) + actual_filepath = message.extra_data.get(_body_param.name, None) _form_data[_body_param.name] = ( - _body_param.type_.__name__ == "UploadFile" + ( + _body_param.type_.__name__ == "UploadFile" + or isinstance(_body_param.field_info, File) + ) + and _file_data is not None + and actual_filepath is not None and UploadFile( file=pathlib.Path(actual_filepath).open("rb"), - size=_file_data["size"], - filename=_file_data["filename"], - headers=Headers({"Content-Type": _file_data["mediaType"]}), + size=_file_data.get("size"), + filename=_file_data.get("filename"), + headers=Headers({"Content-Type": _file_data.get("mediaType")}), ) or _form_data[_body_param.name] ) _verified_args = {} _required_args = getattr(_route.endpoint, "__annotations__", {}) + _signature_args = inspect.signature(_route.endpoint).parameters for arg in _required_args: if arg in _form_data: _verified_args[arg] = _form_data[arg] else: - # needed for /login input - if arg == "authenticated_user": - _verified_args["authenticated_user"] = auth_user + # needed for CleverSwarm /login input if _required_args[arg].__name__ == "OAuth2PasswordRequestForm": try: _verified_args[arg] = OAuth2PasswordRequestForm(**_form_data) except Exception as e: print(f"Error: {e}") + if self.authenticated_user_arg_name in _signature_args.keys(): + _verified_args[self.authenticated_user_arg_name] = auth_user - return await CleverThisServiceAdapter.call_cleverthis_api( + return await self.call_cleverthis_api( message, _verified_args, api_method=_route.endpoint, success_code=_route.status_code if _route.status_code else 200, - loop=self.loop, ) return DataResponseFactory.of( @@ -557,13 +583,98 @@ class CleverThisServiceAdapter: message.trace_info(), ) - @staticmethod + def _verify_and_instantiate_arguments( + self, + args: Any, + api_method: Callable[[Any, Any], Coroutine], + ) -> dict: + """ + Check the provided arguments against the API method signature and convert the incoming string value + into correct expected input object. + The logic used FastAPI annotation and would use the value from 'default_factory' invocation (where defined) + or 'default' value, as defined in the FastAPI annotation, to create default value in case the object is not in + the input values provided by the caller. + + """ + _verified_args = {} + try: + _sig = inspect.signature(api_method) + _required_args = list(_sig.parameters.values()) + # _required_args = getattr(api_method, '__annotations__', {}) + for arg in _required_args: + if arg.name in args: + _annotation = arg.annotation + _str_annotation = str(_annotation) + # convert to another type if arg is not a str + if arg.name == self.authenticated_user_arg_name: + _verified_args[arg.name] = args.get(arg.name) + elif _annotation == int: + _verified_args[arg.name] = int(args.get(arg.name)) + elif _annotation == float: + _verified_args[arg.name] = float(args.get(arg.name)) + elif _annotation == bool: + _verified_args[arg.name] = serializer.str_to_bool(args.get(arg.name)) + elif ( + "Optional" in _str_annotation + or _str_annotation.lower().startswith("list") + or _str_annotation.startswith("typing.List") + or "class" in _str_annotation + ): + _type_descr = TypeDescriptor( + _str_annotation, extra_init_data=g_extra_init_data + ) + try: + _payload = args.get(arg.name, "") + _verified_args[arg.name] = _type_descr.instantiate( + _payload, is_json=is_likely_json(_payload) + ) + except Exception as err: + logging_error( + f"Could not instantiate argument {arg}, defaulting to its string value", + err, + ) + _verified_args[arg.name] = args.get(arg.name, None) + else: + _temp = args.get(arg.name) + if isinstance(arg.annotation, enum.EnumType) or isinstance( + arg.annotation, enum.Enum + ): + try: + _temp = arg.annotation[_temp] + except KeyError as ke: + # some enums have string value different to key spelling, try different thing + try: + _temp = arg.annotation(_temp) + except ValueError as ve: + logging_warning( + f"cannot convert '{_temp}' to Enum type {arg.annotation}, {ke}, {ve}" + ) + print(f"ENUM -> {_temp}") + _verified_args[arg.name] = _temp + else: + if hasattr(arg, "default"): + _temp = arg.default + if hasattr(_temp, "default_factory") and _temp.default_factory is not None: + _temp = _temp.default_factory() + elif hasattr(_temp, "default"): + _temp = _temp.default + if not str(_temp) == "": + _verified_args[arg.name] = _temp + else: + logging_warning( + f"{api_method.__name__}: Missing required argument: {arg.name}" + ) + except Exception as e: + logging_error("Failed to validate and instantiate input args", e) + + return _verified_args + async def call_cleverthis_api( + self, message: AMQMessage, args: Any, api_method: Callable[[Any, Any], Coroutine], success_code: int, - loop: AbstractEventLoop | None = None, ) -> DataResponse: """ Handles POST and PUT requests, which typically contain possibly complex payload data, @@ -578,31 +689,16 @@ class CleverThisServiceAdapter: :return: DataResponse class """ try: - _verified_args = {} - _sig = inspect.signature(api_method) - _required_args = list(_sig.parameters.values()) - # _required_args = getattr(api_method, '__annotations__', {}) - for arg in _required_args: - if arg.name in args: - # convert to another type if arg is not a str - if arg.annotation == int: - _verified_args[arg.name] = int(args[arg.name]) - elif arg.annotation == float: - _verified_args[arg.name] = float(args[arg.name]) - elif arg.annotation == bool: - _verified_args[arg.name] = serializer.str_to_bool(args[arg.name]) - # TODO: more? - else: - _verified_args[arg.name] = args[arg.name] - else: - logging_info(f"{api_method.__name__}: Missing required argument: {arg.name}") + _verified_args = self._verify_and_instantiate_arguments( + args=args, api_method=api_method + ) logging_info( - "INVOKE ENDPOINT: {%s}, ARGS: {%s}", + "INVOKE ENDPOINT: [%s], ARGS: {%s}", api_method.__name__, ", ".join(f"{k}={v}" for k, v in _verified_args.items()), ) _result: Any = await api_method(**_verified_args) - logging_info(f"ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}") + logging_info(f"ENDPOINT RESULT: [{api_method.__name__}], RESULT: {str(_result)}") if isinstance(_result, FileResponse): _json_result, _content_type = ( @@ -610,7 +706,7 @@ class CleverThisServiceAdapter: _result, message.connection, file_handler=StreamingFileHandler(), - loop=loop, + loop=self.loop, ), "application/octet-stream", ) @@ -618,7 +714,10 @@ class CleverThisServiceAdapter: _json_result = _result.json() _content_type = "application/json" else: - _json_result = _result + if isinstance(_result, str): + _json_result = _result + else: + _json_result = python_type_to_json(_result) _content_type = "application/json" _binary_result = ( diff --git a/amqp/adapter/data_parser.py b/amqp/adapter/data_parser.py index 96d718f..46dd9f3 100644 --- a/amqp/adapter/data_parser.py +++ b/amqp/adapter/data_parser.py @@ -51,13 +51,14 @@ class MultipartFormDataParser: ) self.is_multipart = True except Exception as e: - logging_error(f"Error parsing multipart/form-data: {e}. Trying parsing as JSON.") + __error = f"Error parsing multipart/form-data: {e}. Trying parsing as JSON." try: self.form_data = json.loads(message.body()) self.is_json = True if self.form_data["files"] is not None and self.form_data["form"] is not None: self.form_data = AMQDataParser.process_form_data(self.form_data) except Exception as jsonerror: + logging_error(__error) logging_error(f"Parsing JSON: {jsonerror}") self.is_valid = False @@ -148,7 +149,7 @@ class AMQDataParser: # Parse JSON if content_type == "application/json": try: - return json.loads(message.body()) + return json.loads(message.body() or "{}") except json.JSONDecodeError as e: logging_error(f"Invalid JSON: {e}") raise ValueError(f"Invalid JSON: {e}") diff --git a/amqp/adapter/pydantic_serializer.py b/amqp/adapter/pydantic_serializer.py index 27a61ae..61185f4 100644 --- a/amqp/adapter/pydantic_serializer.py +++ b/amqp/adapter/pydantic_serializer.py @@ -5,7 +5,7 @@ This module provides serialization and deserialization functions for Pydantic mo import importlib import json import uuid -from datetime import datetime +from datetime import date, datetime from types import UnionType from typing import Any, Dict, Union, get_args, get_origin from urllib.parse import parse_qs, urlparse @@ -57,6 +57,35 @@ def serialize_object(obj: BaseModel) -> str: return f"{module_name}.{class_name}:{dict_string}" +def python_type_to_json(data): + """ + Convert Python objects (dict, list, tuple, set, datetime, etc.) to a JSON string. + Handles common non-JSON-serializable types like datetime, date, and sets. + + Args: + data: Python object to be converted (dict, list, tuple, set, etc.) + + Returns: + str: JSON string representation of the input data + + Raises: + TypeError: If the object contains non-serializable types (e.g., custom classes) + """ + + def default_serializer(obj): + """Custom serializer for non-JSON-serializable objects.""" + if isinstance(obj, (datetime, date)): + return obj.isoformat() # Convert datetime/date to ISO string + elif isinstance(obj, set): + return list(obj) # Convert sets to lists + elif hasattr(obj, "__dict__"): + return obj.__dict__ # Attempt to serialize custom objects + else: + raise TypeError(f"Object of type {type(obj)} is not JSON serializable") + + return json.dumps(data, default=default_serializer, indent=0, ensure_ascii=False) + + def try_deserialize_object(data: str) -> BaseModel | str: """ Try deserialize object, if failed, return the input string as-is. diff --git a/amqp/adapter/type_descriptor.py b/amqp/adapter/type_descriptor.py new file mode 100644 index 0000000..4fbad7c --- /dev/null +++ b/amqp/adapter/type_descriptor.py @@ -0,0 +1,205 @@ +""" """ + +import importlib +import json +from typing import Any + + +class TypeDescriptor: + def __init__(self, type_string: str, extra_init_data: dict): + self.type_string = type_string + self.extra_init_data = extra_init_data or {} + self.is_json: bool = False + self.type: str = self._extract_type() + + def _extract_type(self) -> str: + if self.type_string.startswith(" 1: + self.is_json = parts[1].lower() == "json" + return parts[0] if parts else "" + else: + # Handle simple Optional case + return annotated + return _extract_type_of(self.type_string) + + def instantiate(self, payload: Any, is_json: bool = False) -> Any: + return _instantiate( + ( + self.type + if "typing.Optional" in self.type_string or self.type_string.startswith(" str: + return f"TypeDescriptor(type='{self.type}', is_json={self.is_json})" + + +def _extract_type_of(input_string: str) -> str: + stack = [] + start_index = -1 + end_index = -1 + + for i, char in enumerate(input_string): + if i > 0: # this assumes paramerized type, that must be at lest 1 char long + if char == "[": + if start_index == -1: + start_index = i + stack.append(char) + elif char == "]": + if len(stack) == 1: + end_index = i + break + if len(stack) > 0: + stack.pop() + + if start_index != -1 and end_index != -1: + return input_string[start_index + 1 : end_index].strip() + else: + return input_string.strip() if start_index == 1 and end_index == -1 else "" + + +def merge_extra_init_data(cls, json_payload, extra_init_data: dict): + """ + Merges extra_init_data into json_payload only when: + 1. cls has an attribute named like the key + 2. json_payload doesn't already have the key + """ + return { + **json_payload, + **{ + k: v + for k, v in extra_init_data.items() + if (hasattr(cls, k) or (hasattr(cls, "model_fields") and k in cls.model_fields)) + and k not in json_payload + }, + } + + +def _instantiate( + type_string: str, payload: Any, is_json: bool = False, extra_init_data: dict = None +) -> Any: + """ + Dynamically imports and instantiates a class from a TypeDescriptor's type_string attribute. + + Args: + type_string: TypeDescriptor instance with the class path + payload: Argument to pass to the constructor + + Returns: + Instance of the target class initialized with payload + + Raises: + ImportError: If the module/class cannot be imported + TypeError: If instantiation fails + """ + # Handle JSON payload first + json_payload = json.loads(payload) if is_json and isinstance(payload, str) else None + + # Handle basic types + if type_string == "str": + return str(payload) + if type_string == "int": + return int(payload) + if type_string == "float": + return float(payload) + if type_string == "bool": + return bool(payload) + + # Handle typing module types + if type_string.startswith("typing.") or type_string.startswith("list"): + if ( + type_string.startswith("typing.List") + or type_string.startswith("typing.Sequence") + or type_string.startswith("list") + ): + if json_payload is not None: + return json_payload + inner_type = _extract_type_of(type_string) + if inner_type: + return [ + _instantiate(inner_type, p, extra_init_data=extra_init_data) + for p in (payload if isinstance(payload, list) else payload.split(",")) + ] + return payload if isinstance(payload, list) else payload.split(",") + + if type_string.startswith("typing.Dict") or type_string.startswith("typing.Mapping"): + if json_payload is not None: + return json_payload + return dict(payload) + + if type_string.startswith("typing.Set"): + if json_payload is not None: + return set(json_payload) + _set_type = _extract_type_of(type_string) + return set( + payload + if isinstance(payload, list) + else [_instantiate(_set_type, p) for p in payload.split(",")] + ) + + if type_string.startswith("typing.Tuple"): + if json_payload is not None: + return tuple(json_payload) + _tuple_type = _extract_type_of(type_string) + return tuple( + payload + if isinstance(payload, list) + else [_instantiate(_tuple_type, p) for p in payload.split(",")] + ) + + if type_string.startswith("typing.Optional"): + inner_type = _extract_type_of(type_string) + if inner_type: + return _instantiate(inner_type, payload, is_json, extra_init_data) + return payload + + # Handle regular class instantiation + parts = type_string.split(".") + module_path = ".".join(parts[:-1]) + class_name = parts[-1] + + try: + # Import the module + module = importlib.import_module(module_path) + # Get the class + cls = getattr(module, class_name) + if isinstance(payload, cls): + return payload + # Instantiate with payload + if json_payload is not None: + _args = merge_extra_init_data(cls, json_payload, extra_init_data or {}) + if hasattr(cls, "from_dict"): + return cls.from_dict(_args) + elif hasattr(cls, "create"): + return cls.create(**_args) + return cls(**_args) + + if isinstance(payload, dict): + _args = merge_extra_init_data(cls, payload, extra_init_data or {}) + if hasattr(cls, "from_dict"): + return cls.from_dict(_args) + elif hasattr(cls, "create"): + return cls.create(**_args) + return cls(**_args) + + return cls(payload) + except ImportError as e: + raise ImportError(f"Could not import module '{module_path}'") from e + except AttributeError as e: + raise ImportError(f"Module '{module_path}' has no class '{class_name}'") from e + except TypeError as e: + raise TypeError(f"Failed to instantiate {type_string} with payload {payload}") from e diff --git a/cleverbrag/clever_brag_adapter.py b/cleverbrag/clever_brag_adapter.py index 47112b7..c5cceef 100644 --- a/cleverbrag/clever_brag_adapter.py +++ b/cleverbrag/clever_brag_adapter.py @@ -104,8 +104,11 @@ class CleverBragAmqpAdapter(CleverThisServiceAdapter): amq_configuration: AMQConfiguration, routes: List[APIRoute], auth_provider: core.base.providers.auth.AuthProvider, + extra_init_data: dict, ): - super().__init__(amq_configuration, routes, None) + super().__init__( + amq_configuration, routes, None, "auth_user", extra_init_data=extra_init_data + ) self.auth_provider = auth_provider _amq_service: AMQService = AMQService(amq_configuration, self) self.thread = Thread(target=_amq_service.run) diff --git a/cleverbrag/main.py b/cleverbrag/main.py index e8f97a6..5bf9fef 100644 --- a/cleverbrag/main.py +++ b/cleverbrag/main.py @@ -21,7 +21,10 @@ if __name__ == "__main__": r2rapp: R2RApp = create_r2r_app() adapter: CleverBragAmqpAdapter = CleverBragAmqpAdapter( - AMQConfiguration("./application.properties.local"), r2rapp.app.routes, r2rapp.providers.auth + AMQConfiguration("./application.properties.local"), + r2rapp.app.routes, + r2rapp.providers.auth, + {"app": r2rapp.config.app}, ) for method, routes in adapter.routes.items(): diff --git a/cleverbrag/tests/test_collections_router.sh b/cleverbrag/tests/test_collections_router.sh new file mode 100644 index 0000000..0d0c08f --- /dev/null +++ b/cleverbrag/tests/test_collections_router.sh @@ -0,0 +1,149 @@ +echo "#!/bin/bash" + +. ./test_shared_functions.sh + +extract_document_id() { + R=$1 + C=$(echo $R | sed 's/[0-9][0-9][0-9]$//') + #echo ">>${C}<<" + if [[ ! -z "$C" ]] && [[ "$C" == *results* ]]; then + COLLECTION_IDS=($(echo $C | jq -r '.results[] | select(.document_count >= 1) | .id' | tr '\n' ' ')) + echo "CIDs = ${COLLECTION_IDS[*]}" + else + echo "No IDs detected in the response." + fi + HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/') +} + +echo "# 1. Create Collection Tests" +echo "=== Testing Collection Creation ===" + +echo "# Test 1.1: Create collection with name only" +make_request "POST" "/collections" \ + "-d '{\"name\":\"Test Collection\"}'" + +echo "# Test 1.2: Create collection with name and description" +make_request "POST" "/collections" \ + "-d '{\"name\":\"Test Collection\",\"description\":\"A test collection\"}'" + +echo "# Test 1.3: Create collection with empty name (should fail)" +make_request "POST" "/collections" \ + "-d '{\"name\":\"\"}'" + +echo "# Test 1.4: Create collection with very long name (edge case)" +make_request "POST" "/collections" \ + "-d '{\"name\":\"$(printf 'a%.0s' {1..1000})\"}'" + +echo "# 2. List Collections Tests" +echo "=== Testing Collection Listing ===" + +echo "# Test 2.1: List all collections" +make_request "GET" "/collections" "" +ALL_COLLECTION_IDS=("${COLLECTION_IDS[@]}") + +echo "# Test 2.2: List collections with pagination" +make_request "GET" "/collections?offset=0&limit=10" "" + +echo "# Test 2.3: List collections with invalid pagination (edge cases)" +make_request "GET" "/collections?offset=-1&limit=10" "" +make_request "GET" "/collections?offset=0&limit=0" "" +make_request "GET" "/collections?offset=0&limit=1001" "" + +echo "# Test 2.4: List specific collections by IDs" +# Join array elements with commas +COMMA_SEPARATED_IDS=$(echo "${ALL_COLLECTION_IDS[*]}" | sed -e "s/ /,/") +make_request "GET" "/collections?ids=${COMMA_SEPARATED_IDS}" "" +echo "# 3. Get Collection Tests" +echo "=== Testing Collection Retrieval ===" + +echo "# Test 3.1: Get collection by ID" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}" "" + +echo "# Test 3.2: Get non-existent collection" +make_request "GET" "/collections/00000000-0000-0000-0000-000000000000" "" + +echo "# Test 3.3: Get collection with invalid UUID format" +make_request "GET" "/collections/invalid-uuid" "" + +echo "# 4. Update Collection Tests" +echo "=== Testing Collection Updates ===" + +echo "# Test 4.1: Update collection name" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \ + "-d '{\"name\":\"Updated Collection Name\"}'" + +echo "# Test 4.2: Update collection description" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \ + "-d '{\"description\":\"Updated description\"}'" + +echo "# Test 4.3: Update both name and description" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \ + "-d '{\"name\":\"Updated Name\",\"description\":\"Updated description\"}'" + +echo "# Test 4.4: Generate description" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \ + "-d '{\"generate_description\":true}'" + +echo "# Test 4.5: Invalid update (both description and generate_description)" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \ + "-d '{\"description\":\"New description\",\"generate_description\":true}'" + +echo "# 5. Delete Collection Tests" +echo "=== Testing Collection Deletion ===" + +echo "# Test 5.1: Delete existing collection" +make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}" "" + +echo "# Test 5.2: Delete non-existent collection" +make_request "DELETE" "/collections/00000000-0000-0000-0000-000000000000" "" + +echo "# 6. Document Management Tests" +echo "=== Testing Document Management ===" + +echo "# Test 6.1: Add document to collection" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" "" + +echo "# Test 6.2: List documents in collection" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=10" "" + +echo "# Test 6.3: List documents with invalid pagination" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=-1&limit=10" "" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=1001" "" + +echo "# Test 6.4: Remove document from collection" +make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" "" + +echo "# 7. User Management Tests" +echo "=== Testing User Management ===" + +echo "# Test 7.1: List users in collection" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=10" "" + +echo "# Test 7.2: List users with invalid pagination" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=-1&limit=10" "" +make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=1001" "" + +echo "# Test 7.3: Add user to collection" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" "" + +echo "# Test 7.4: Remove user from collection" +make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" "" + +echo "# 8. Entity Extraction Tests" +echo "=== Testing Entity Extraction ===" + +echo "# Test 8.1: Extract entities with default settings" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \ + "-d '{\"run_type\":\"RUN\"}'" + +echo "# Test 8.2: Extract entities with custom settings" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \ + "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'" + +echo "# Test 8.3: Extract entities without orchestration" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \ + "-d '{\"run_type\":\"RUN\",\"run_with_orchestration\":false}'" + +echo "# Test 8.4: Get estimate only" +make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \ +"-d '{\"run_type\":\"ESTIMATE\"}'" diff --git a/cleverbrag/tests/test_data/AMERICA-VENTURE_HIGHWAY.html b/cleverbrag/tests/test_data/AMERICA-VENTURE_HIGHWAY.html new file mode 100644 index 0000000..f6e32a4 --- /dev/null +++ b/cleverbrag/tests/test_data/AMERICA-VENTURE_HIGHWAY.html @@ -0,0 +1,1130 @@ + + + + AMERICA - VENTURE HIGHWAY LYRICS + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + + + +
+ + + +
+ +
+
+ + + +
+ Like +
+ + +
+
+ +
+
+ +
+ +
+ + + + + +
+ + +
+ +
+ +
+ +
+

America - Venture Highway Lyrics

+

Artist: America

+

Album: Capital Gold Summer Legends

+ +
+ Heyo! SONGLYRICS just got interactive. Highlight. Review: RIFF-it.
RIFF-it good. +
+
+ + + + + + + + + + +
+ +
+ +
+ + + + +
+

Chewing on a piece of grass
+walking down the road
+tell me, how long you gonna stay here joe?
+some people say this town don't look
+good in snow
+you don't care, i know
+Ventura highway in the sunshine
+where the days are longer
+the nights are stronger
+than moonshine
+you're gonna go i know
+'cause the free wind is blowin' through
+your hair
+and the days surround your daylight
+there
+seasons crying no despair
+alligator lizards in the air
+Wishin' on a falling star
+watchin' for the early train
+sorry boy, but i've been hit by
+purple rain
+aw, come on joe, you can always
+change your name
+thanks a lot son, just the same
+Ventura highway
+where the days are longer
+the nights are stronger
+than moonshine
+you're gonna go i know
+'cause the free wind is blowin' through
+your hair
+and the days surround you daylight
+there
+seasons crying no despair
+alligator lizards in the air

+
+
+ + +
+ + +
+
+ + +
+
+ + + + +
+
+ +
    +
  • +
    • +
+ + + + +
Submit lyrics correction →
+
+ +
+ +
+
+ + + + +
+ +
+
+ + + +
+ Like +
+
+ + +
+ +
+ +
+ +
+ +
+ +
+ +
+ + + + +

Add Comment

+ +
+ +
+ + + +
+ +
+ +
+ +
+ +
+ +
+
+ + +
+
+ + +
+
+ +
+ +
+ +
+
+ + + + +
+

In the Know

+ + + + + + + +
+   +

+ All Music News » +
+ + +
+

Popular America Lyrics

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
1All I Think About Is You
2A Horse with No Name (From "American Hustle")
3Story of a Teenager (Rehearsal - The Record Plant, Sausalito, Ca, 1975)
4Look up, Look Down (Unreleased Demo - Buzz Studios, King’s Road, Los Angeles, Ca, 1973)
5Cornwall Blank (Early Rehearsal - The Record Plant, Los Angeles, Ca, 1972)
6Stephen Bishop / Separate Lives
+
+ + + +
+ + + + +
+ +
+ +
+ +
+ + + + +
+ +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + + + + + + + + + + + + + + +
+

Genres

+ + 		+

Top Lyrics

+ + 		+

SongLyrics

+ + 		+

Company

+ +
+ + + +
+ + + + + + + + + + + +
+ + + +
+
+
+ Lead RIFFs: +
+
+ +
+ +
+
Bad selection
+
+

Cannot annotate a non-flat selection. Make sure your selection + starts and ends within the same node.

+
+ (example of bad selection): This is bold + text and this is normal text. +
+
+ (example of good selection): This is bold + text and this is normal text. +
+
+
+ +
+
Bad selection
+
+

An annotation cannot contain another annotation.

+
+
+ + + + + +
+
+ + +
+
Anonymous
+ + + + +
+ + +
+ + +
+ +
+ + +
+
+ + +
+
+ Really delete this comment? +
+ + +
+
+
+ + +
+ +
+ +
+ + +
+ + + +
+ +
+ + + + +
+ + +
+ + +
+
Anonymous
+ + + + +
+ + +
+ + +
+ +
+ + +
+
+ + +
+
+ Really delete this comment? +
+ + +
+
+
+ +
+ +
+ +
+ + +
+ + + +
+ +
+ +
+ + + + + + + + + + + + diff --git a/cleverbrag/tests/test_data/America-Horse_with_no_name.txt b/cleverbrag/tests/test_data/America-Horse_with_no_name.txt new file mode 100644 index 0000000..66b9cb1 --- /dev/null +++ b/cleverbrag/tests/test_data/America-Horse_with_no_name.txt @@ -0,0 +1,42 @@ +On the first part of the journey +I was looking at all the life +There were plants and birds and rocks and things +There was sand and hills and rings +The first thing I met was a fly with a buzz +And the sky with no clouds +The heat was hot and the ground was dry +But the air was full of sound + +I've been through the desert on a horse with no name +It felt good to be out of the rain +In the desert you can remember your name +'Cause there ain't no one for to give you no pain +La, la ... + +After two days in the desert sun +My skin began to turn red +After three days in the desert fun +I was looking at a river bed +And the story it told of a river that flowed +Made me sad to think it was dead + +You see I've been through the desert on a horse with no name +It felt good to be out of the rain +In the desert you can remember your name +'Cause there ain't no one for to give you no pain +La, la ... + +After nine days I let the horse run free +'Cause the desert had turned to sea +There were plants and birds and rocks and things +there was sand and hills and rings +The ocean is a desert with it's life underground +And a perfect disguise above +Under the cities lies a heart made of ground +But the humans will give no love + +You see I've been through the desert on a horse with no name +It felt good to be out of the rain +In the desert you can remember your name +'Cause there ain't no one for to give you no pain +La, la ... diff --git a/cleverbrag/tests/test_data/China_Beijing_Summer_Palace2.jpg b/cleverbrag/tests/test_data/China_Beijing_Summer_Palace2.jpg new file mode 100644 index 0000000..9d48256 Binary files /dev/null and b/cleverbrag/tests/test_data/China_Beijing_Summer_Palace2.jpg differ diff --git a/cleverbrag/tests/test_data/GAR_ARchitecture.odg b/cleverbrag/tests/test_data/GAR_ARchitecture.odg new file mode 100644 index 0000000..54cdf31 Binary files /dev/null and b/cleverbrag/tests/test_data/GAR_ARchitecture.odg differ diff --git a/cleverbrag/tests/test_data/Instituto_Superior_Tecnico.md.json b/cleverbrag/tests/test_data/Instituto_Superior_Tecnico.md.json new file mode 100644 index 0000000..1d11438 --- /dev/null +++ b/cleverbrag/tests/test_data/Instituto_Superior_Tecnico.md.json @@ -0,0 +1 @@ +[{"@id": "data:text/plain;charset=US-ASCII,instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#Organization"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,loures", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,Rogerio%20colaco", "@type": "http://www.cleverthis.com/university#Person"}, {"@id": "data:text/plain;charset=US-ASCII,1049-001%20lisboa", "@type": "data:,"}, {"@id": "data:text/plain;charset=US-ASCII,Instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#University", "http://www.cleverthis.com/university#campus": {"@id": "data:text/plain;charset=US-ASCII,three%20campuses", "@type": "http://www.cleverthis.com/university#Campus"}, "data:,": {"@id": "data:text/plain;charset=US-ASCII,tecnico", "@type": "http://www.cleverthis.com/university#Name"}, "http://www.cleverthis.com/university#country": {"@id": "data:text/plain;charset=US-ASCII,portugal", "@type": "http://www.cleverthis.com/university#Country"}}, {"@id": "data:text/plain;charset=US-ASCII,taguspark", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,oeiras", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,alameda", "@type": "http://www.cleverthis.com/university#Campus"}] diff --git a/cleverbrag/tests/test_data/Worldbank-Datasheet-02082022.xlsx b/cleverbrag/tests/test_data/Worldbank-Datasheet-02082022.xlsx new file mode 100644 index 0000000..73221c8 Binary files /dev/null and b/cleverbrag/tests/test_data/Worldbank-Datasheet-02082022.xlsx differ diff --git a/cleverbrag/tests/test_data/flight_of_the_bumblebee_2.mp3 b/cleverbrag/tests/test_data/flight_of_the_bumblebee_2.mp3 new file mode 100644 index 0000000..52262c2 Binary files /dev/null and b/cleverbrag/tests/test_data/flight_of_the_bumblebee_2.mp3 differ diff --git a/cleverbrag/tests/test_documents_router.sh b/cleverbrag/tests/test_documents_router.sh new file mode 100644 index 0000000..dd68887 --- /dev/null +++ b/cleverbrag/tests/test_documents_router.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +. ./test_shared_functions.sh + +JSON_FILE=./test_data/Instituto_Superior_Tecnico.md.json +HTML_FILE=./test_data/AMERICA-VENTURE_HIGHWAY.html +MP3_FILE=./test_data/flight_of_the_bumblebee_2.mp3 +JPG_FILE=./test_data/China_Beijing_Summer_Palace2.jpg +EXCEL_FILE=./test_data/Worldbank-Datasheet-02082022.xlsx +TEXT_FILE=./test_data/America-Horse_with_no_name.txt +OPEN_OFFICE_FILE=./test_data/GAR_ARchitecture.odg + +echo "# 1. Create Document Tests" +echo "=== Testing Document Creation ===" + +echo "# Test 1.1: Create document with file, use different formats" +make_request "POST" "/documents" \ + "-F file=@${TEXT_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${JSON_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_JSON_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${HTML_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_HTML_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${MP3_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_MP3_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${JPG_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_JPG_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${EXCEL_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_EXCEL_FILE=${DOCUMENT_ID} + +make_request "POST" "/documents" \ + "-F file=@${OPEN_OFFICE_FILE}" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_OPEN_OFFICE_FILE=${DOCUMENT_ID} + +echo "# Test 1.2: Create document with raw text" +make_request "POST" "/documents" \ + '-F raw_text="On the first part of the journey +I was looking at all the life +There were plants and birds and rocks and things +There was sand and hills and rings +The first thing I met was a fly with a buzz +And the sky with no clouds +The heat was hot and the ground was dry +But the air was full of sound"' \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_RAW_TEXT=${DOCUMENT_ID} + +echo "# Test 1.3: Create document with chunks" +make_request "POST" "/documents" \ + "-F chunks=\"[\\\"I've been through the desert on a horse with no name\\\",\\\"It felt good to be out of the rain\\\"]\"" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_CHUNKS=${DOCUMENT_ID} + +echo "# Test 1.4: Create document with metadata" +make_request "POST" "/documents" \ + "-F file=@${TEXT_FILE} -F metadata='{\"title\":\"Horse with No Name\",\"author\":\"America\"}'" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_FILE_WITH_METADATA=${DOCUMENT_ID} + +echo "# Test 1.5: Create document with custom ingestion config" +make_request "POST" "/documents" \ + "-F file=@${TEXT_FILE} -F ingestion_mode=hi-res -F ingestion_config='{\"chunk_size\":1000}'" \ + "-H 'Content-Type: multipart/form-data'" +DOCUMENT_ID_FROM_FILE_WITH_CUSTOM_INGEST=${DOCUMENT_ID} + +echo "# 2. List Documents Tests" +echo "=== Testing Document Listing ===" + +echo "# Test 2.1: List all documents" +make_request "GET" "/documents" "" + +echo "# Test 2.2: List documents with pagination" +make_request "GET" "/documents?offset=0&limit=10" "" + +echo "# Test 2.3: List specific documents by IDs" +make_request "GET" "/documents?ids=${DOCUMENT_ID_FROM_CHUNKS},${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" "" + +echo "# Test 2.4: List documents with summary embeddings" +make_request "GET" "/documents?include_summary_embeddings=1" "" + +echo "# 3. Get Document Tests" +echo "=== Testing Document Retrieval ===" + +echo "# Test 3.1: Get document by ID" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" "" + +echo "# Test 3.2: Get non-existent document" +make_request "GET" "/documents/00000000-0000-0000-0000-000000000000" "" + +echo "# 4. List Document Chunks Tests" +echo "=== Testing Document Chunks ===" + +echo "# Test 4.1: List chunks with pagination" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?offset=0&limit=10" "" + +echo "# Test 4.2: List chunks with vectors" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?include_vectors=true" "" + +echo "# 5. Download Document Tests" +echo "=== Testing Document Download ===" + +echo "# Test 5.1: Download document" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE}/download" "" + +echo "# 6. Delete Document Tests" +echo "=== Testing Document Deletion ===" + +echo "# Test 6.1: Delete document by ID" +make_request "DELETE" "/documents/${DOCUMENT_ID_FROM_RAW_TEXT}" "" + +echo "# Test 6.2: Delete documents by filter" +make_request "DELETE" "/documents/by-filter" \ + "-d '{\"document_type\":{\"\$eq\":\"txt\"}}'" + +echo "# 7. Document Collections Tests" +echo "=== Testing Document Collections ===" + +echo "# Test 7.1: List document collections" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/collections?offset=0&limit=10" "" + +echo "# 8. Entity Extraction Tests" +echo "=== Testing Entity Extraction ===" + +echo "# Test 8.1: Extract entities with default settings" +make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \ + "-d '{\"run_type\":\"RUN\"}'" + +echo "# Test 8.2: Extract entities with custom settings" +make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \ + "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'" + +echo "# 9. Entity Retrieval Tests" +echo "=== Testing Entity Retrieval ===" + +echo "# Test 9.1: Get entities with pagination" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?offset=0&limit=10" "" + +echo "# Test 9.2: Get entities with embeddings" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?include_embeddings=true" "" + +echo "# 10. Relationship Tests" +echo "=== Testing Relationships ===" + +echo "# Test 10.1: Get relationships with pagination" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?offset=0&limit=10" "" + +echo "# Test 10.2: Get relationships with filters" +make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?entity_names=person,company&relationship_types=works_for" "" + +echo "# 11. Document Search Tests" +echo "=== Testing Document Search ===" + +echo "# Test 11.1: Basic search" +make_request "POST" "/documents/search" \ + "-d '{\"query\":\"test query\",\"search_mode\":\"basic\"}'" + +echo "# Test 11.2: Advanced search with custom settings" +make_request "POST" "/documents/search" \ + "-d '{\"query\":\"test query\",\"search_mode\":\"advanced\",\"search_settings\":{\"filters\":{\"document_type\":\"pdf\"}}}'" + +echo "# Test 11.3: Custom search with complex filters" +make_request "POST" "/documents/search" \ + "-d '{\"query\":\"test query\",\"search_mode\":\"custom\",\"search_settings\":{\"filters\":{\"created_at\":{\"\$gt\":\"2023-01-01\"}}}}'" diff --git a/cleverbrag/tests/test_shared_functions.sh b/cleverbrag/tests/test_shared_functions.sh new file mode 100644 index 0000000..9566571 --- /dev/null +++ b/cleverbrag/tests/test_shared_functions.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# +# Generates random valid UUID +# +generate_uuid() { + local N B C='89ab' + local RES='' + for (( N=0; N < 16; ++N )) do + B=$(( $RANDOM % 256 )) + + case $N in + 6) + RES="${RES}$(printf '4%x' $(( B % 16 )))" + ;; + 8) + RES="${RES}$(printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B % 16 )))" + ;; + 3 | 5 | 7 | 9) + RES="${RES}$(printf '%02x-' $B)" + ;; + *) + RES="${RES}$(printf '%02x' $B)" + ;; + esac + done + echo -n $RES +} + +# +# Extract document_id (if present) and HTTP response code from curl response passed as argument +# +extract_document_id() { + DOCUMENT_ID=$(echo $1 | grep document_id | tail -1 | sed -e 's/^.*"document_id": *"\([^"]*\)".*$/\1/') + HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/') +} + +# Helper function to make curl requests +make_request() { + local method=$1 + local endpoint=$2 + local data=$3 + local headers=$4 + if [ "xx${headers}" == "xx" ]; then + headers='-H "Content-Type: application/json"' + fi + # echo "Testing $method $endpoint" + # echo "Request data: $data" + # echo "Request headers: $headers" + CMD="curl -X $method \"$API_BASE$endpoint\" -s -w \"\n%{http_code}\" -H \"Authorization: Bearer $AUTH_TOKEN\" $headers $data" + echo $CMD + RES=$(eval $CMD 2>/dev/null) + echo "Response: [${RES}]" + extract_document_id "$RES" + # echo "document_id --> ${DOCUMENT_ID}" + if [[ ${HTTP_RETURN_CODE} -lt 300 ]]; then + echo "SUCCESS. code = ${HTTP_RETURN_CODE}" + else + echo "ERROR. code = ${HTTP_RETURN_CODE}" + fi + echo -e "\n\n" + if [ "${EXECUTE_ALL}" != "-a" ]; then + echo "Press any key to continue..." + read -n 1 -s -r + echo -e "\n\n" + fi +} + +# +# Simple scanner of cmd line arguments, inferring meaning of the value +# It recognized: +# a) host URL (starts with 'http' prefix) +# b) JWT token (starts with 'ey' prefix) +# c) flag -a to execute all tests w/o waiting for user pressing any key +# +while [ $# -gt 0 ]; do + echo "Argument: [${1}]" + if [[ "$1" == ey* ]]; then + AUTH_TOKEN=$1 + fi + if [[ "$1" == http* ]]; then + HOST=$1 + fi + if [[ "$1" == "-a" ]]; then + EXECUTE_ALL=$1 + fi + shift # Removes $1 and shifts remaining args left +done + + +if [ "xx${HOST}" == "xx" ]; then + HOST="http://localhost:7272" +fi +API_BASE="$HOST/v3" +# Once integrated with CleverMicro, token can be obtained from CleverMicro +# if [ "xx${AUTH_TOKEN}" == "xx" ]; then +# AUTH_TOKEN=$(curl $HOST/api/v0/login -s -d '{"username":"hugo@cleverthis.com","password":"tellmeall"}' -H "Content-Type: application/json" | sed -e "s/.*access_token...\(.*\)..$/\1/") +# fi diff --git a/pyproject.toml b/pyproject.toml index 36d8b7f..79de392 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "amq_adapter" -version = "0.2.32" +version = "0.2.33" description = "The Python implementation of the AMQ Adapter for CleverMicro" readme = "README.md" authors = [ diff --git a/tests/adapter/test_backpressure_handler.py b/tests/adapter/test_backpressure_handler.py index 73482e0..42b7219 100644 --- a/tests/adapter/test_backpressure_handler.py +++ b/tests/adapter/test_backpressure_handler.py @@ -284,7 +284,7 @@ class TestBackpressureHandler: assert handler.config == mock_config assert handler.swarm_service_id == "clever-amqp-service" assert handler.swarm_task_id == "python-adapter" - assert handler.parallel_workers == 1 + assert handler.parallel_workers == mock_config.backpressure.threshold_threads assert handler.average_cpu_usage is None def test_increase_parallel_executions(self, mock_channel, mock_loop, mock_config): diff --git a/tests/adapter/test_cleverthis_service_adapter.py b/tests/adapter/test_cleverthis_service_adapter.py index e3688c4..daca052 100644 --- a/tests/adapter/test_cleverthis_service_adapter.py +++ b/tests/adapter/test_cleverthis_service_adapter.py @@ -1,10 +1,13 @@ import asyncio import unittest +from enum import Enum +from typing import Any, List, Optional from unittest.mock import AsyncMock, MagicMock, patch from aio_pika.abc import AbstractChannel, AbstractRobustConnection from aiohttp import ClientResponse from aiohttp.web_fileresponse import FileResponse +from pydantic import BaseModel from amqp.adapter.cleverthis_service_adapter import ( AMQMessage, @@ -19,6 +22,26 @@ from amqp.model.model import DataResponse from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE +class NestedModel(BaseModel): + id: int + name: str + + +class ComplexModel(BaseModel): + nested: NestedModel + items: List[str] + + +class TestEnum(Enum): + VALUE1 = "value1" + VALUE2 = "value2" + + +class TestModel(BaseModel): + name: str + value: int + + class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase): async def test_create_reply_exchange_and_queue_success(self): @@ -89,6 +112,7 @@ class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase): amq_configuration=self.mock_amq_config, routes=[], session=self.mock_session ) self.adapter.service_host = "testhost" + self.adapter.authenticated_user_arg_name = "user" def test_init(self): self.assertEqual(self.adapter.service_host, "testhost") @@ -282,6 +306,126 @@ class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase): with self.assertRaises(NotImplementedError): await self.adapter.head(mock_message, None) + def test_basic_type_conversion(self): + async def test_method(x: int, y: float, z: bool, text: str): + pass + + args = {"x": "123", "y": "45.67", "z": "true", "text": "hello"} + + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["x"], 123) + self.assertEqual(result["y"], 45.67) + self.assertEqual(result["z"], True) + self.assertEqual(result["text"], "hello") + + def test_enum_conversion(self): + async def test_method(status: TestEnum): + pass + + args = {"status": "value1"} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["status"], TestEnum.VALUE1) + + def test_optional_parameter(self): + async def test_method(required: str, optional: Optional[str] = None): + pass + + args = {"required": "test"} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["required"], "test") + self.assertIsNone(result["optional"]) + + def test_list_parameter(self): + async def test_method(items: List[str]): + pass + + args = {"items": "item1,item2,item3"} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["items"], ["item1", "item2", "item3"]) + + def test_pydantic_model_parameter(self): + async def test_method(model: TestModel): + pass + + args = {"model": '{"name": "test", "value": 42}'} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertIsInstance(result["model"], TestModel) + self.assertEqual(result["model"].name, "test") + self.assertEqual(result["model"].value, 42) + + def test_authenticated_user_parameter(self): + async def test_method(user: Any): + pass + + user_obj = MagicMock() + args = {"user": user_obj} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["user"], user_obj) + + def test_missing_required_parameter(self): + async def test_method(required: str): + pass + + args = {} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertNotIn("required", result) + + def test_invalid_enum_value(self): + async def test_method(status: TestEnum): + pass + + args = {"status": "invalid_value"} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["status"], "invalid_value") + + def test_invalid_type_conversion(self): + async def test_method(x: int): + pass + + args = {"x": "not_a_number"} + result = {} + try: + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + except Exception as e: + self.assertTrue(isinstance(e, TypeError)) + + self.assertEqual(len(result), 0) + + def test_complex_nested_model(self): + + async def test_method(model: ComplexModel): + pass + + args = {"model": {"nested": {"id": 1, "name": "test"}, "items": ["item1", "item2"]}} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertIsInstance(result["model"], ComplexModel) + self.assertEqual(result["model"].nested.id, 1) + self.assertEqual(result["model"].nested.name, "test") + self.assertEqual(result["model"].items, ["item1", "item2"]) + + def test_default_factory_parameter(self): + class DefaultFactory: + @staticmethod + def default_factory(): + return "default_value" + + async def test_method(param: str = DefaultFactory()): + pass + + args = {} + result = self.adapter._verify_and_instantiate_arguments(args, test_method) + + self.assertEqual(result["param"], "default_value") + if __name__ == "__main__": unittest.main() diff --git a/tests/adapter/test_pydantic_serializer.py b/tests/adapter/test_pydantic_serializer.py index af2b9ff..266536a 100644 --- a/tests/adapter/test_pydantic_serializer.py +++ b/tests/adapter/test_pydantic_serializer.py @@ -1,4 +1,5 @@ -from datetime import datetime +import json +from datetime import date, datetime from unittest import TestCase from uuid import UUID @@ -7,6 +8,7 @@ from pydantic import BaseModel from amqp.adapter.pydantic_serializer import ( deserialize_object, parse_url_query, + python_type_to_json, serialize_object, ) @@ -130,3 +132,95 @@ class PydanticSerializerTest(TestCase): ) self.assertEqual(path, "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741") self.assertEqual(args, {}) + + +class TestPythonTypeToJson: + """ + Test cases for the python_type_to_json function. + """ + + def test_basic_types(self): + """Test serialization of basic Python types.""" + # Test dictionary + data = {"key": "value", "number": 42} + result = python_type_to_json(data) + assert json.loads(result) == data + + # Test list + data = [1, 2, 3, "four"] + result = python_type_to_json(data) + assert json.loads(result) == data + + # Test tuple + data = (1, 2, 3) + result = python_type_to_json(data) + assert json.loads(result) == [1, 2, 3] # Tuples are converted to lists + + # Test set + data = {1, 2, 3} + result = python_type_to_json(data) + assert set(json.loads(result)) == data # Sets are converted to lists + + def test_datetime_handling(self): + """Test serialization of datetime objects.""" + # Test datetime + test_datetime = datetime(2024, 1, 1, 12, 0, 0) + data = {"timestamp": test_datetime} + result = python_type_to_json(data) + assert json.loads(result) == {"timestamp": test_datetime.isoformat()} + + # Test date + test_date = date(2024, 1, 1) + data = {"date": test_date} + result = python_type_to_json(data) + assert json.loads(result) == {"date": test_date.isoformat()} + + def test_nested_structures(self): + """Test serialization of nested data structures.""" + data = { + "list_of_datetimes": [datetime(2024, 1, 1), datetime(2024, 1, 2)], + "dict_with_date": {"date": date(2024, 1, 1)}, + "mixed": {"set": {1, 2, 3}, "tuple": (1, 2, 3), "datetime": datetime(2024, 1, 1)}, + } + result = python_type_to_json(data) + parsed = json.loads(result) + assert parsed["list_of_datetimes"] == [d.isoformat() for d in data["list_of_datetimes"]] + assert parsed["dict_with_date"]["date"] == data["dict_with_date"]["date"].isoformat() + assert set(parsed["mixed"]["set"]) == data["mixed"]["set"] + assert parsed["mixed"]["tuple"] == list(data["mixed"]["tuple"]) + assert parsed["mixed"]["datetime"] == data["mixed"]["datetime"].isoformat() + + def test_custom_object_serialization(self): + """Test serialization of custom objects with __dict__ attribute.""" + + class TestObject: + def __init__(self, value): + self.value = value + self.timestamp = datetime(2024, 1, 1) + + obj = TestObject("test") + data = {"obj": obj} + result = python_type_to_json(data) + parsed = json.loads(result) + assert parsed["obj"]["value"] == "test" + assert parsed["obj"]["timestamp"] == obj.timestamp.isoformat() + + def test_empty_structures(self): + """Test serialization of empty data structures.""" + # Empty dict + assert python_type_to_json({}) == "{}" + + # Empty list + assert python_type_to_json([]) == "[]" + + # Empty set + assert python_type_to_json(set()) == "[]" + + # Empty tuple + assert python_type_to_json(()) == "[]" + + def test_unicode_handling(self): + """Test handling of Unicode characters.""" + data = {"text": "Hello, 世界!"} + result = python_type_to_json(data) + assert json.loads(result) == data # ensure_ascii=False should preserve Unicode diff --git a/tests/adapter/test_service_message_factory.py b/tests/adapter/test_service_message_factory.py index 72ab467..6561000 100644 --- a/tests/adapter/test_service_message_factory.py +++ b/tests/adapter/test_service_message_factory.py @@ -1,9 +1,9 @@ from unittest import TestCase from amqp.adapter.service_message_factory import ( - ServiceMessageFactory, - RS, INVALID_MESSAGE, + RS, + ServiceMessageFactory, ) from amqp.model.model import ServiceMessage from amqp.model.service_message_type import ServiceMessageType @@ -13,15 +13,11 @@ class ServiceMessageTest(TestCase): def test_to_string(self): _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee") - message: ServiceMessage = _f.of( - ServiceMessageType.ROUTING_DATA, "Duck", "Donald" - ) + message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald") message_str = _f.to_string(message) self.assertTrue("|type=01|" in message_str) self.assertTrue("|body=Duck" in message_str) - message: ServiceMessage = _f.of( - ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald" - ) + message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald") message_str = _f.to_string(message) self.assertTrue("|type=03|" in message_str) self.assertTrue("|body=Duck" in message_str) @@ -57,3 +53,4 @@ class ServiceMessageTest(TestCase): message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald") serialized = _f.serialize(message) assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value) + assert len(serialized) > 16 diff --git a/tests/adapter/test_type_descriptor.py b/tests/adapter/test_type_descriptor.py new file mode 100644 index 0000000..c37f3eb --- /dev/null +++ b/tests/adapter/test_type_descriptor.py @@ -0,0 +1,211 @@ +import json +import unittest + +from pydantic import BaseModel + +from amqp.adapter.type_descriptor import TypeDescriptor, _instantiate + + +class TestClass(BaseModel): + value: str + extra: str + + +class TestTypeDescriptor(unittest.TestCase): + def test_annotated_with_json(self): + type_str = ( + "typing.Optional[typing.Annotated[core.base.providers.ingestion.IngestionConfig, Json]]" + ) + td = TypeDescriptor(type_str, {}) + + self.assertEqual(td.type, "core.base.providers.ingestion.IngestionConfig") + self.assertTrue(td.is_json) + + def test_annotated_without_json(self): + type_str = "typing.Optional[typing.Annotated[str, SomethingElse]]" + td = TypeDescriptor(type_str, {}) + + self.assertEqual(td.type, "str") + self.assertFalse(td.is_json) + + def test_annotated_multiple_args(self): + type_str = "typing.Optional[typing.Annotated[list[int], Json, more_args]]" + td = TypeDescriptor(type_str, {}) + + self.assertEqual(td.type, "list[int]") + self.assertTrue(td.is_json) + + def test_simple_optional(self): + type_str = "typing.Optional[uuid.UUID]" + td = TypeDescriptor(type_str, {}) + + self.assertEqual(td.type, "uuid.UUID") + self.assertFalse(td.is_json) + + def test_whitespace_variations(self): + test_cases = [ + ("typing.Optional[typing.Annotated[ str , Json ]]", "str", True), + ("typing.Optional[ typing.Annotated[dict,Json]]", "dict", True), + ("typing.Optional[ int ]", "int", False), + ] + + for type_str, expected_type, expected_json in test_cases: + with self.subTest(type_str=type_str): + td = TypeDescriptor(type_str, {}) + self.assertEqual(td.type, expected_type) + self.assertEqual(td.is_json, expected_json) + + def test_invalid_input(self): + test_cases = [ + "typing.Optional[missing_bracket", + "typing.Optional[]", + "[without.optional]", + ] + + for type_str in test_cases: + with self.subTest(type_str=type_str): + td = TypeDescriptor(type_str, {}) + self.assertEqual(td.type, "") + self.assertFalse(td.is_json) + + def test_repr(self): + type_str = "typing.Optional[typing.Annotated[str, Json]]" + td = TypeDescriptor(type_str, {}) + + self.assertEqual(repr(td), "TypeDescriptor(type='str', is_json=True)") + + def test_instantiate_basic_types(self): + test_cases = [ + ("str", "hello", "hello"), + ("int", "42", 42), + ("float", "3.14", 3.14), + ("bool", "true", True), + ] + + for type_str, input_value, expected in test_cases: + with self.subTest(type_str=type_str, input_value=input_value): + result = _instantiate(type_str, input_value) + self.assertEqual(result, expected) + + def test_instantiate_list_types(self): + test_cases = [ + ("typing.List[str]", "a,b,c", ["a", "b", "c"]), + ("typing.List[int]", "1,2,3", [1, 2, 3]), + ("typing.List[str]", json.dumps(["a", "b", "c"]), ["a", "b", "c"], True), + ("typing.List[int]", json.dumps([1, 2, 3]), [1, 2, 3], True), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_dict_types(self): + test_cases = [ + ("typing.Dict", {"a": 1, "b": 2}, {"a": 1, "b": 2}), + ("typing.Dict", json.dumps({"a": 1, "b": 2}), {"a": 1, "b": 2}, True), + ("typing.Mapping", {"a": 1, "b": 2}, {"a": 1, "b": 2}), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_set_types(self): + test_cases = [ + ("typing.Set[str]", "a,b,c", {"a", "b", "c"}), + ("typing.Set[int]", "1,2,3", {1, 2, 3}), + ("typing.Set[str]", json.dumps(["a", "b", "c"]), {"a", "b", "c"}, True), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_tuple_types(self): + test_cases = [ + ("typing.Tuple[str, int]", ["hello", 42], ("hello", 42)), + ("typing.Tuple[str, int]", json.dumps(["hello", 42]), ("hello", 42), True), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_optional_types(self): + test_cases = [ + ("typing.Optional[str]", "hello", "hello"), + ("typing.Optional[typing.List[str]]", "a,b,c", ["a", "b", "c"]), + ( + "typing.Optional[typing.List[str]]", + json.dumps(["a", "b", "c"]), + ["a", "b", "c"], + True, + ), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_nested_types(self): + test_cases = [ + ( + "typing.List[typing.Dict]", + json.dumps([{"a": 1}, {"b": 2}]), + [{"a": 1}, {"b": 2}], + True, + ), + ( + "typing.Optional[typing.List[typing.Dict]]", + json.dumps([{"a": 1}, {"b": 2}]), + [{"a": 1}, {"b": 2}], + True, + ), + ] + + for type_str, input_value, expected, *args in test_cases: + is_json = args[0] if args else False + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + result = _instantiate(type_str, input_value, is_json=is_json) + self.assertEqual(result, expected) + + def test_instantiate_error_handling(self): + test_cases = [ + ("typing.List[str]", "invalid json", True, json.JSONDecodeError), + ("typing.NonExistentType", "value", False, ImportError), + ] + + for type_str, input_value, is_json, expected_exception in test_cases: + with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json): + with self.assertRaises(expected_exception): + _instantiate(type_str, input_value, is_json=is_json) + + def test_instantiate_with_extra_init_data(self): + + test_cases = [ + ( + "tests.adapter.test_type_descriptor.TestClass", + {"value": "test"}, + {"extra": "extra_value"}, + lambda x: x.value == "test" and x.extra == "extra_value", + ), + ] + + for type_str, input_value, extra_init_data, validator in test_cases: + with self.subTest(type_str=type_str, input_value=input_value): + result = _instantiate(type_str, input_value, extra_init_data=extra_init_data) + self.assertTrue(validator(result)) + + +if __name__ == "__main__": + unittest.main()