Merge branch 'cleverbrag-integration-2' into develop

amqp/adapter/cleverthis_service_adapter.py

@@ -4,6 +4,7 @@ Methods here will invoke the actual CleverThis Service code, which are passed as
 """
 
 import asyncio
+import enum
 import inspect
 import json
 import pathlib
@@ -19,6 +20,7 @@ from aio_pika.abc import (
 )
 from aiohttp import ClientResponse, ClientSession
 from fastapi import HTTPException, UploadFile
+from fastapi.params import File
 from fastapi.routing import APIRoute
 from fastapi.security import OAuth2PasswordRequestForm
 from pydantic import BaseModel
@@ -29,12 +31,21 @@ from amqp.adapter import serializer
 from amqp.adapter.data_parser import AMQDataParser
 from amqp.adapter.data_response_factory import DataResponseFactory
 from amqp.adapter.file_handler import FileHandler, StreamingFileHandler
-from amqp.adapter.logging_utils import logging_debug, logging_error, logging_info
+from amqp.adapter.logging_utils import (
+    logging_debug,
+    logging_error,
+    logging_info,
+    logging_warning,
+)
+from amqp.adapter.pydantic_serializer import python_type_to_json
+from amqp.adapter.type_descriptor import TypeDescriptor
 from amqp.config.amq_configuration import AMQConfiguration
 from amqp.model.model import AMQErrorMessage, AMQMessage, DataMessage, DataResponse
 from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
 from amqp.router.utils import await_result
 
+g_extra_init_data: dict = dict()
+
 
 async def _create_reply_exchange_and_queue(
     connection: AbstractRobustConnection,
@@ -100,6 +111,8 @@ async def _convert_file_response(
 
 def is_likely_json(text: str) -> bool:
     """Quick test to see if should convert the response string to JSON format."""
+    if not isinstance(text, str):
+        return False
     _text = text.strip()
     return (_text.startswith("{") and _text.endswith("}")) or (
         _text.startswith("[") and _text.endswith("]")
@@ -120,12 +133,17 @@ class CleverThisServiceAdapter:
         amq_configuration: AMQConfiguration,
         routes: List[APIRoute],
         session: ClientSession = None,
+        authenticated_user_arg_name: str = "authenticated_user",
+        extra_init_data=None,
     ):
         """
         Initializes the CleverThisServiceAdapter with the given AMQ configuration and optional HTTP
         session. The HTTP session is for the case when the adapter uses loose coupling with
         the CleverThis service via HTTP REST API.
         """
+        if extra_init_data is None:
+            extra_init_data = dict()
+        self.authenticated_user_arg_name = authenticated_user_arg_name
         self.amq_configuration = amq_configuration
         self.session = session
         self.service_host = self.amq_configuration.dispatch.service_host
@@ -135,6 +153,8 @@ class CleverThisServiceAdapter:
             self.amq_configuration.amq_adapter.require_authenticated_user
         )
         self.routes = self.group_routes_by_method(api_routes=routes)
+        global g_extra_init_data
+        g_extra_init_data = extra_init_data or {}
 
     def get_endpoint(self, endpoint: Any) -> Any:
         """
@@ -474,14 +494,15 @@ class CleverThisServiceAdapter:
             _match = _route.path_regex.match(_path)
             if _match:
                 _args.update(_match.groupdict())
-                _args["authenticated_user"] = auth_user
+                _signature_args = inspect.signature(_route.endpoint).parameters
+                if self.authenticated_user_arg_name in _signature_args.keys():
+                    _args[self.authenticated_user_arg_name] = auth_user
                 # If the path matches, call the corresponding function
-                return await CleverThisServiceAdapter.call_cleverthis_api(
+                return await self.call_cleverthis_api(
                     message,
                     _args,
                     api_method=_route.endpoint,
                     success_code=_route.status_code,
-                    loop=self.loop,
                 )
 
         return DataResponseFactory.of(
@@ -513,40 +534,45 @@ class CleverThisServiceAdapter:
                 _form_data.update(_query_args)
                 for _body_param in _route.dependant.body_params:
                     if _body_param.name in _form_data:
-                        _file_data = _form_data[_body_param.name]
-                        actual_filepath = message.extra_data[_body_param.name]
+                        _file_data = _form_data.get(_body_param.name, None)
+                        actual_filepath = message.extra_data.get(_body_param.name, None)
                         _form_data[_body_param.name] = (
+                            (
                                 _body_param.type_.__name__ == "UploadFile"
+                                or isinstance(_body_param.field_info, File)
+                            )
+                            and _file_data is not None
+                            and actual_filepath is not None
                             and UploadFile(
                                 file=pathlib.Path(actual_filepath).open("rb"),
-                                size=_file_data["size"],
-                                filename=_file_data["filename"],
-                                headers=Headers({"Content-Type": _file_data["mediaType"]}),
+                                size=_file_data.get("size"),
+                                filename=_file_data.get("filename"),
+                                headers=Headers({"Content-Type": _file_data.get("mediaType")}),
                             )
                             or _form_data[_body_param.name]
                         )
 
                 _verified_args = {}
                 _required_args = getattr(_route.endpoint, "__annotations__", {})
+                _signature_args = inspect.signature(_route.endpoint).parameters
                 for arg in _required_args:
                     if arg in _form_data:
                         _verified_args[arg] = _form_data[arg]
                     else:
-                        # needed for /login  input
-                        if arg == "authenticated_user":
-                            _verified_args["authenticated_user"] = auth_user
+                        # needed for CleverSwarm /login  input
                         if _required_args[arg].__name__ == "OAuth2PasswordRequestForm":
                             try:
                                 _verified_args[arg] = OAuth2PasswordRequestForm(**_form_data)
                             except Exception as e:
                                 print(f"Error: {e}")
+                if self.authenticated_user_arg_name in _signature_args.keys():
+                    _verified_args[self.authenticated_user_arg_name] = auth_user
 
-                return await CleverThisServiceAdapter.call_cleverthis_api(
+                return await self.call_cleverthis_api(
                     message,
                     _verified_args,
                     api_method=_route.endpoint,
                     success_code=_route.status_code if _route.status_code else 200,
-                    loop=self.loop,
                 )
 
         return DataResponseFactory.of(
@@ -557,13 +583,98 @@ class CleverThisServiceAdapter:
             message.trace_info(),
         )
 
-    @staticmethod
+    def _verify_and_instantiate_arguments(
+        self,
+        args: Any,
+        api_method: Callable[[Any, Any], Coroutine],
+    ) -> dict:
+        """
+        Check the provided arguments against the API method signature and convert the incoming string value
+        into correct expected input object.
+        The logic used FastAPI annotation and would use the value from 'default_factory' invocation (where defined)
+        or 'default' value, as defined in the FastAPI annotation, to create default value in case the object is not in
+        the input values provided by the caller.
+
+        """
+        _verified_args = {}
+        try:
+            _sig = inspect.signature(api_method)
+            _required_args = list(_sig.parameters.values())
+            # _required_args = getattr(api_method, '__annotations__', {})
+            for arg in _required_args:
+                if arg.name in args:
+                    _annotation = arg.annotation
+                    _str_annotation = str(_annotation)
+                    # convert to another type if arg is not a str
+                    if arg.name == self.authenticated_user_arg_name:
+                        _verified_args[arg.name] = args.get(arg.name)
+                    elif _annotation == int:
+                        _verified_args[arg.name] = int(args.get(arg.name))
+                    elif _annotation == float:
+                        _verified_args[arg.name] = float(args.get(arg.name))
+                    elif _annotation == bool:
+                        _verified_args[arg.name] = serializer.str_to_bool(args.get(arg.name))
+                    elif (
+                        "Optional" in _str_annotation
+                        or _str_annotation.lower().startswith("list")
+                        or _str_annotation.startswith("typing.List")
+                        or "class" in _str_annotation
+                    ):
+                        _type_descr = TypeDescriptor(
+                            _str_annotation, extra_init_data=g_extra_init_data
+                        )
+                        try:
+                            _payload = args.get(arg.name, "")
+                            _verified_args[arg.name] = _type_descr.instantiate(
+                                _payload, is_json=is_likely_json(_payload)
+                            )
+                        except Exception as err:
+                            logging_error(
+                                f"Could not instantiate argument {arg}, defaulting to its string value",
+                                err,
+                            )
+                            _verified_args[arg.name] = args.get(arg.name, None)
+                    else:
+                        _temp = args.get(arg.name)
+                        if isinstance(arg.annotation, enum.EnumType) or isinstance(
+                            arg.annotation, enum.Enum
+                        ):
+                            try:
+                                _temp = arg.annotation[_temp]
+                            except KeyError as ke:
+                                # some enums have string value different to key spelling, try different thing
+                                try:
+                                    _temp = arg.annotation(_temp)
+                                except ValueError as ve:
+                                    logging_warning(
+                                        f"cannot convert '{_temp}' to Enum type {arg.annotation}, {ke}, {ve}"
+                                    )
+                            print(f"ENUM -> {_temp}")
+                        _verified_args[arg.name] = _temp
+                else:
+                    if hasattr(arg, "default"):
+                        _temp = arg.default
+                        if hasattr(_temp, "default_factory") and _temp.default_factory is not None:
+                            _temp = _temp.default_factory()
+                        elif hasattr(_temp, "default"):
+                            _temp = _temp.default
+                        if not str(_temp) == "<class 'inspect._empty'>":
+                            _verified_args[arg.name] = _temp
+                    else:
+                        logging_warning(
+                            f"{api_method.__name__}: Missing required argument: {arg.name}"
+                        )
+        except Exception as e:
+            logging_error("Failed to validate and instantiate input args", e)
+
+        return _verified_args
+
     async def call_cleverthis_api(
+        self,
         message: AMQMessage,
         args: Any,
         api_method: Callable[[Any, Any], Coroutine],
         success_code: int,
-        loop: AbstractEventLoop | None = None,
     ) -> DataResponse:
         """
         Handles POST and PUT requests, which typically contain possibly complex payload data,
@@ -578,31 +689,16 @@ class CleverThisServiceAdapter:
         :return: DataResponse class
         """
         try:
-            _verified_args = {}
-            _sig = inspect.signature(api_method)
-            _required_args = list(_sig.parameters.values())
-            # _required_args = getattr(api_method, '__annotations__', {})
-            for arg in _required_args:
-                if arg.name in args:
-                    # convert to another type if arg is not a str
-                    if arg.annotation == int:
-                        _verified_args[arg.name] = int(args[arg.name])
-                    elif arg.annotation == float:
-                        _verified_args[arg.name] = float(args[arg.name])
-                    elif arg.annotation == bool:
-                        _verified_args[arg.name] = serializer.str_to_bool(args[arg.name])
-                    # TODO: more?
-                    else:
-                        _verified_args[arg.name] = args[arg.name]
-                else:
-                    logging_info(f"{api_method.__name__}: Missing required argument: {arg.name}")
+            _verified_args = self._verify_and_instantiate_arguments(
+                args=args, api_method=api_method
+            )
             logging_info(
-                "INVOKE ENDPOINT: {%s}, ARGS: {%s}",
+                "INVOKE ENDPOINT: [%s], ARGS: {%s}",
                 api_method.__name__,
                 ", ".join(f"{k}={v}" for k, v in _verified_args.items()),
             )
             _result: Any = await api_method(**_verified_args)
-            logging_info(f"ENDPOINT RESULT: {api_method.__name__}, RESULT: {str(_result)}")
+            logging_info(f"ENDPOINT RESULT: [{api_method.__name__}], RESULT: {str(_result)}")
 
             if isinstance(_result, FileResponse):
                 _json_result, _content_type = (
@@ -610,7 +706,7 @@ class CleverThisServiceAdapter:
                         _result,
                         message.connection,
                         file_handler=StreamingFileHandler(),
-                        loop=loop,
+                        loop=self.loop,
                     ),
                     "application/octet-stream",
                 )
@@ -618,7 +714,10 @@ class CleverThisServiceAdapter:
                 _json_result = _result.json()
                 _content_type = "application/json"
             else:
+                if isinstance(_result, str):
                     _json_result = _result
+                else:
+                    _json_result = python_type_to_json(_result)
                 _content_type = "application/json"
 
             _binary_result = (

amqp/adapter/data_parser.py

@@ -51,13 +51,14 @@ class MultipartFormDataParser:
                 )
                 self.is_multipart = True
         except Exception as e:
-            logging_error(f"Error parsing multipart/form-data: {e}. Trying parsing as JSON.")
+            __error = f"Error parsing multipart/form-data: {e}. Trying parsing as JSON."
             try:
                 self.form_data = json.loads(message.body())
                 self.is_json = True
                 if self.form_data["files"] is not None and self.form_data["form"] is not None:
                     self.form_data = AMQDataParser.process_form_data(self.form_data)
             except Exception as jsonerror:
+                logging_error(__error)
                 logging_error(f"Parsing JSON: {jsonerror}")
                 self.is_valid = False
 
@@ -148,7 +149,7 @@ class AMQDataParser:
         # Parse JSON
         if content_type == "application/json":
             try:
-                return json.loads(message.body())
+                return json.loads(message.body() or "{}")
             except json.JSONDecodeError as e:
                 logging_error(f"Invalid JSON: {e}")
                 raise ValueError(f"Invalid JSON: {e}")

amqp/adapter/pydantic_serializer.py

@@ -5,7 +5,7 @@ This module provides serialization and deserialization functions for Pydantic mo
 import importlib
 import json
 import uuid
-from datetime import datetime
+from datetime import date, datetime
 from types import UnionType
 from typing import Any, Dict, Union, get_args, get_origin
 from urllib.parse import parse_qs, urlparse
@@ -57,6 +57,35 @@ def serialize_object(obj: BaseModel) -> str:
     return f"{module_name}.{class_name}:{dict_string}"
 
 
+def python_type_to_json(data):
+    """
+    Convert Python objects (dict, list, tuple, set, datetime, etc.) to a JSON string.
+    Handles common non-JSON-serializable types like datetime, date, and sets.
+
+    Args:
+        data: Python object to be converted (dict, list, tuple, set, etc.)
+
+    Returns:
+        str: JSON string representation of the input data
+
+    Raises:
+        TypeError: If the object contains non-serializable types (e.g., custom classes)
+    """
+
+    def default_serializer(obj):
+        """Custom serializer for non-JSON-serializable objects."""
+        if isinstance(obj, (datetime, date)):
+            return obj.isoformat()  # Convert datetime/date to ISO string
+        elif isinstance(obj, set):
+            return list(obj)  # Convert sets to lists
+        elif hasattr(obj, "__dict__"):
+            return obj.__dict__  # Attempt to serialize custom objects
+        else:
+            raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+    return json.dumps(data, default=default_serializer, indent=0, ensure_ascii=False)
+
+
 def try_deserialize_object(data: str) -> BaseModel | str:
     """
     Try deserialize object, if failed, return the input string as-is.

amqp/adapter/type_descriptor.py

@@ -0,0 +1,205 @@
+""" """
+
+import importlib
+import json
+from typing import Any
+
+
+class TypeDescriptor:
+    def __init__(self, type_string: str, extra_init_data: dict):
+        self.type_string = type_string
+        self.extra_init_data = extra_init_data or {}
+        self.is_json: bool = False
+        self.type: str = self._extract_type()
+
+    def _extract_type(self) -> str:
+        if self.type_string.startswith("<class '"):
+            return self.type_string[8:-2]
+        # Handle Annotated case of FastAPI annotation
+        if "typing.Optional" in self.type_string:
+            annotated = _extract_type_of(self.type_string)
+            if "typing.Annotated" in annotated:
+                # Extract content between the first square brackets after Annotated
+                match = _extract_type_of(annotated)
+                if match:
+                    # Split by comma and get first part, then strip whitespace
+                    parts = [p.strip() for p in match.split(",")]
+                    if len(parts) > 1:
+                        self.is_json = parts[1].lower() == "json"
+                    return parts[0] if parts else ""
+            else:
+                # Handle simple Optional case
+                return annotated
+        return _extract_type_of(self.type_string)
+
+    def instantiate(self, payload: Any, is_json: bool = False) -> Any:
+        return _instantiate(
+            (
+                self.type
+                if "typing.Optional" in self.type_string or self.type_string.startswith("<class ")
+                else self.type_string
+            ),
+            payload,
+            self.is_json or is_json,
+            extra_init_data=self.extra_init_data,
+        )
+
+    def __repr__(self) -> str:
+        return f"TypeDescriptor(type='{self.type}', is_json={self.is_json})"
+
+
+def _extract_type_of(input_string: str) -> str:
+    stack = []
+    start_index = -1
+    end_index = -1
+
+    for i, char in enumerate(input_string):
+        if i > 0:  # this assumes paramerized type, that must be at lest 1 char long
+            if char == "[":
+                if start_index == -1:
+                    start_index = i
+                stack.append(char)
+            elif char == "]":
+                if len(stack) == 1:
+                    end_index = i
+                    break
+                if len(stack) > 0:
+                    stack.pop()
+
+    if start_index != -1 and end_index != -1:
+        return input_string[start_index + 1 : end_index].strip()
+    else:
+        return input_string.strip() if start_index == 1 and end_index == -1 else ""
+
+
+def merge_extra_init_data(cls, json_payload, extra_init_data: dict):
+    """
+    Merges extra_init_data into json_payload only when:
+    1. cls has an attribute named like the key
+    2. json_payload doesn't already have the key
+    """
+    return {
+        **json_payload,
+        **{
+            k: v
+            for k, v in extra_init_data.items()
+            if (hasattr(cls, k) or (hasattr(cls, "model_fields") and k in cls.model_fields))
+            and k not in json_payload
+        },
+    }
+
+
+def _instantiate(
+    type_string: str, payload: Any, is_json: bool = False, extra_init_data: dict = None
+) -> Any:
+    """
+    Dynamically imports and instantiates a class from a TypeDescriptor's type_string attribute.
+
+    Args:
+        type_string: TypeDescriptor instance with the class path
+        payload: Argument to pass to the constructor
+
+    Returns:
+        Instance of the target class initialized with payload
+
+    Raises:
+        ImportError: If the module/class cannot be imported
+        TypeError: If instantiation fails
+    """
+    # Handle JSON payload first
+    json_payload = json.loads(payload) if is_json and isinstance(payload, str) else None
+
+    # Handle basic types
+    if type_string == "str":
+        return str(payload)
+    if type_string == "int":
+        return int(payload)
+    if type_string == "float":
+        return float(payload)
+    if type_string == "bool":
+        return bool(payload)
+
+    # Handle typing module types
+    if type_string.startswith("typing.") or type_string.startswith("list"):
+        if (
+            type_string.startswith("typing.List")
+            or type_string.startswith("typing.Sequence")
+            or type_string.startswith("list")
+        ):
+            if json_payload is not None:
+                return json_payload
+            inner_type = _extract_type_of(type_string)
+            if inner_type:
+                return [
+                    _instantiate(inner_type, p, extra_init_data=extra_init_data)
+                    for p in (payload if isinstance(payload, list) else payload.split(","))
+                ]
+            return payload if isinstance(payload, list) else payload.split(",")
+
+        if type_string.startswith("typing.Dict") or type_string.startswith("typing.Mapping"):
+            if json_payload is not None:
+                return json_payload
+            return dict(payload)
+
+        if type_string.startswith("typing.Set"):
+            if json_payload is not None:
+                return set(json_payload)
+            _set_type = _extract_type_of(type_string)
+            return set(
+                payload
+                if isinstance(payload, list)
+                else [_instantiate(_set_type, p) for p in payload.split(",")]
+            )
+
+        if type_string.startswith("typing.Tuple"):
+            if json_payload is not None:
+                return tuple(json_payload)
+            _tuple_type = _extract_type_of(type_string)
+            return tuple(
+                payload
+                if isinstance(payload, list)
+                else [_instantiate(_tuple_type, p) for p in payload.split(",")]
+            )
+
+        if type_string.startswith("typing.Optional"):
+            inner_type = _extract_type_of(type_string)
+            if inner_type:
+                return _instantiate(inner_type, payload, is_json, extra_init_data)
+            return payload
+
+    # Handle regular class instantiation
+    parts = type_string.split(".")
+    module_path = ".".join(parts[:-1])
+    class_name = parts[-1]
+
+    try:
+        # Import the module
+        module = importlib.import_module(module_path)
+        # Get the class
+        cls = getattr(module, class_name)
+        if isinstance(payload, cls):
+            return payload
+        # Instantiate with payload
+        if json_payload is not None:
+            _args = merge_extra_init_data(cls, json_payload, extra_init_data or {})
+            if hasattr(cls, "from_dict"):
+                return cls.from_dict(_args)
+            elif hasattr(cls, "create"):
+                return cls.create(**_args)
+            return cls(**_args)
+
+        if isinstance(payload, dict):
+            _args = merge_extra_init_data(cls, payload, extra_init_data or {})
+            if hasattr(cls, "from_dict"):
+                return cls.from_dict(_args)
+            elif hasattr(cls, "create"):
+                return cls.create(**_args)
+            return cls(**_args)
+
+        return cls(payload)
+    except ImportError as e:
+        raise ImportError(f"Could not import module '{module_path}'") from e
+    except AttributeError as e:
+        raise ImportError(f"Module '{module_path}' has no class '{class_name}'") from e
+    except TypeError as e:
+        raise TypeError(f"Failed to instantiate {type_string} with payload {payload}") from e

cleverbrag/clever_brag_adapter.py

@@ -104,8 +104,11 @@ class CleverBragAmqpAdapter(CleverThisServiceAdapter):
         amq_configuration: AMQConfiguration,
         routes: List[APIRoute],
         auth_provider: core.base.providers.auth.AuthProvider,
+        extra_init_data: dict,
     ):
-        super().__init__(amq_configuration, routes, None)
+        super().__init__(
+            amq_configuration, routes, None, "auth_user", extra_init_data=extra_init_data
+        )
         self.auth_provider = auth_provider
         _amq_service: AMQService = AMQService(amq_configuration, self)
         self.thread = Thread(target=_amq_service.run)

cleverbrag/main.py

@@ -21,7 +21,10 @@ if __name__ == "__main__":
     r2rapp: R2RApp = create_r2r_app()
 
     adapter: CleverBragAmqpAdapter = CleverBragAmqpAdapter(
-        AMQConfiguration("./application.properties.local"), r2rapp.app.routes, r2rapp.providers.auth
+        AMQConfiguration("./application.properties.local"),
+        r2rapp.app.routes,
+        r2rapp.providers.auth,
+        {"app": r2rapp.config.app},
     )
 
     for method, routes in adapter.routes.items():

cleverbrag/tests/test_collections_router.sh

@@ -0,0 +1,149 @@
+echo "#!/bin/bash"
+
+. ./test_shared_functions.sh
+
+extract_document_id() {
+  R=$1
+  C=$(echo $R | sed 's/[0-9][0-9][0-9]$//')
+  #echo ">>${C}<<"
+  if [[ ! -z "$C" ]] && [[ "$C" == *results* ]]; then
+    COLLECTION_IDS=($(echo $C | jq -r '.results[] | select(.document_count >= 1) | .id' | tr '\n' ' '))
+    echo "CIDs = ${COLLECTION_IDS[*]}"
+  else
+    echo "No IDs detected in the response."
+  fi
+  HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
+}
+
+echo "# 1. Create Collection Tests"
+echo "=== Testing Collection Creation ==="
+
+echo "# Test 1.1: Create collection with name only"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"Test Collection\"}'"
+
+echo "# Test 1.2: Create collection with name and description"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"Test Collection\",\"description\":\"A test collection\"}'"
+
+echo "# Test 1.3: Create collection with empty name (should fail)"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"\"}'"
+
+echo "# Test 1.4: Create collection with very long name (edge case)"
+make_request "POST" "/collections" \
+    "-d '{\"name\":\"$(printf 'a%.0s' {1..1000})\"}'"
+
+echo "# 2. List Collections Tests"
+echo "=== Testing Collection Listing ==="
+
+echo "# Test 2.1: List all collections"
+make_request "GET" "/collections" ""
+ALL_COLLECTION_IDS=("${COLLECTION_IDS[@]}")
+
+echo "# Test 2.2: List collections with pagination"
+make_request "GET" "/collections?offset=0&limit=10" ""
+
+echo "# Test 2.3: List collections with invalid pagination (edge cases)"
+make_request "GET" "/collections?offset=-1&limit=10" ""
+make_request "GET" "/collections?offset=0&limit=0" ""
+make_request "GET" "/collections?offset=0&limit=1001" ""
+
+echo "# Test 2.4: List specific collections by IDs"
+# Join array elements with commas
+COMMA_SEPARATED_IDS=$(echo "${ALL_COLLECTION_IDS[*]}" | sed -e "s/ /,/")
+make_request "GET" "/collections?ids=${COMMA_SEPARATED_IDS}" ""
+echo "# 3. Get Collection Tests"
+echo "=== Testing Collection Retrieval ==="
+
+echo "# Test 3.1: Get collection by ID"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}" ""
+
+echo "# Test 3.2: Get non-existent collection"
+make_request "GET" "/collections/00000000-0000-0000-0000-000000000000" ""
+
+echo "# Test 3.3: Get collection with invalid UUID format"
+make_request "GET" "/collections/invalid-uuid" ""
+
+echo "# 4. Update Collection Tests"
+echo "=== Testing Collection Updates ==="
+
+echo "# Test 4.1: Update collection name"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"name\":\"Updated Collection Name\"}'"
+
+echo "# Test 4.2: Update collection description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"description\":\"Updated description\"}'"
+
+echo "# Test 4.3: Update both name and description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"name\":\"Updated Name\",\"description\":\"Updated description\"}'"
+
+echo "# Test 4.4: Generate description"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"generate_description\":true}'"
+
+echo "# Test 4.5: Invalid update (both description and generate_description)"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}" \
+    "-d '{\"description\":\"New description\",\"generate_description\":true}'"
+
+echo "# 5. Delete Collection Tests"
+echo "=== Testing Collection Deletion ==="
+
+echo "# Test 5.1: Delete existing collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}" ""
+
+echo "# Test 5.2: Delete non-existent collection"
+make_request "DELETE" "/collections/00000000-0000-0000-0000-000000000000" ""
+
+echo "# 6. Document Management Tests"
+echo "=== Testing Document Management ==="
+
+echo "# Test 6.1: Add document to collection"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
+
+echo "# Test 6.2: List documents in collection"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=10" ""
+
+echo "# Test 6.3: List documents with invalid pagination"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=-1&limit=10" ""
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/documents?offset=0&limit=1001" ""
+
+echo "# Test 6.4: Remove document from collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/documents/456e789a-b12c-34d5-e678-901234567890" ""
+
+echo "# 7. User Management Tests"
+echo "=== Testing User Management ==="
+
+echo "# Test 7.1: List users in collection"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=10" ""
+
+echo "# Test 7.2: List users with invalid pagination"
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=-1&limit=10" ""
+make_request "GET" "/collections/${ALL_COLLECTION_IDS[0]}/users?offset=0&limit=1001" ""
+
+echo "# Test 7.3: Add user to collection"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
+
+echo "# Test 7.4: Remove user from collection"
+make_request "DELETE" "/collections/${ALL_COLLECTION_IDS[0]}/users/789a012b-c34d-5e6f-a789-012345678901" ""
+
+echo "# 8. Entity Extraction Tests"
+echo "=== Testing Entity Extraction ==="
+
+echo "# Test 8.1: Extract entities with default settings"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\"}'"
+
+echo "# Test 8.2: Extract entities with custom settings"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
+
+echo "# Test 8.3: Extract entities without orchestration"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"run_with_orchestration\":false}'"
+
+echo "# Test 8.4: Get estimate only"
+make_request "POST" "/collections/${ALL_COLLECTION_IDS[0]}/extract" \
+"-d '{\"run_type\":\"ESTIMATE\"}'"

cleverbrag/tests/test_data/America-Horse_with_no_name.txt

@@ -0,0 +1,42 @@
+On the first part of the journey
+I was looking at all the life
+There were plants and birds and rocks and things
+There was sand and hills and rings
+The first thing I met was a fly with a buzz
+And the sky with no clouds
+The heat was hot and the ground was dry
+But the air was full of sound
+
+I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...
+
+After two days in the desert sun
+My skin began to turn red
+After three days in the desert fun
+I was looking at a river bed
+And the story it told of a river that flowed
+Made me sad to think it was dead
+
+You see I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...
+
+After nine days I let the horse run free
+'Cause the desert had turned to sea
+There were plants and birds and rocks and things
+there was sand and hills and rings
+The ocean is a desert with it's life underground
+And a perfect disguise above
+Under the cities lies a heart made of ground
+But the humans will give no love
+
+You see I've been through the desert on a horse with no name
+It felt good to be out of the rain
+In the desert you can remember your name
+'Cause there ain't no one for to give you no pain
+La, la ...

cleverbrag/tests/test_data/Instituto_Superior_Tecnico.md.json

@@ -0,0 +1 @@
+[{"@id": "data:text/plain;charset=US-ASCII,instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#Organization"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,lisbon", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,loures", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,Rogerio%20colaco", "@type": "http://www.cleverthis.com/university#Person"}, {"@id": "data:text/plain;charset=US-ASCII,1049-001%20lisboa", "@type": "data:,"}, {"@id": "data:text/plain;charset=US-ASCII,Instituto%20superior%20tecnico", "@type": "http://www.cleverthis.com/university#University", "http://www.cleverthis.com/university#campus": {"@id": "data:text/plain;charset=US-ASCII,three%20campuses", "@type": "http://www.cleverthis.com/university#Campus"}, "data:,": {"@id": "data:text/plain;charset=US-ASCII,tecnico", "@type": "http://www.cleverthis.com/university#Name"}, "http://www.cleverthis.com/university#country": {"@id": "data:text/plain;charset=US-ASCII,portugal", "@type": "http://www.cleverthis.com/university#Country"}}, {"@id": "data:text/plain;charset=US-ASCII,taguspark", "@type": "http://www.cleverthis.com/university#Campus"}, {"@id": "data:text/plain;charset=US-ASCII,oeiras", "@type": "http://www.cleverthis.com/university#City"}, {"@id": "data:text/plain;charset=US-ASCII,alameda", "@type": "http://www.cleverthis.com/university#Campus"}]

cleverbrag/tests/test_documents_router.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+
+. ./test_shared_functions.sh
+
+JSON_FILE=./test_data/Instituto_Superior_Tecnico.md.json
+HTML_FILE=./test_data/AMERICA-VENTURE_HIGHWAY.html
+MP3_FILE=./test_data/flight_of_the_bumblebee_2.mp3
+JPG_FILE=./test_data/China_Beijing_Summer_Palace2.jpg
+EXCEL_FILE=./test_data/Worldbank-Datasheet-02082022.xlsx
+TEXT_FILE=./test_data/America-Horse_with_no_name.txt
+OPEN_OFFICE_FILE=./test_data/GAR_ARchitecture.odg
+
+echo "# 1. Create Document Tests"
+echo "=== Testing Document Creation ==="
+
+echo "# Test 1.1: Create document with file, use different formats"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${JSON_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_JSON_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${HTML_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_HTML_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${MP3_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_MP3_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${JPG_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_JPG_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${EXCEL_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_EXCEL_FILE=${DOCUMENT_ID}
+
+make_request "POST" "/documents" \
+    "-F file=@${OPEN_OFFICE_FILE}" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_OPEN_OFFICE_FILE=${DOCUMENT_ID}
+
+echo "# Test 1.2: Create document with raw text"
+make_request "POST" "/documents" \
+    '-F raw_text="On the first part of the journey
+I was looking at all the life
+There were plants and birds and rocks and things
+There was sand and hills and rings
+The first thing I met was a fly with a buzz
+And the sky with no clouds
+The heat was hot and the ground was dry
+But the air was full of sound"' \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_RAW_TEXT=${DOCUMENT_ID}
+
+echo "# Test 1.3: Create document with chunks"
+make_request "POST" "/documents" \
+    "-F chunks=\"[\\\"I've been through the desert on a horse with no name\\\",\\\"It felt good to be out of the rain\\\"]\"" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_CHUNKS=${DOCUMENT_ID}
+
+echo "# Test 1.4: Create document with metadata"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE} -F metadata='{\"title\":\"Horse with No Name\",\"author\":\"America\"}'" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE_WITH_METADATA=${DOCUMENT_ID}
+
+echo "# Test 1.5: Create document with custom ingestion config"
+make_request "POST" "/documents" \
+    "-F file=@${TEXT_FILE} -F ingestion_mode=hi-res -F ingestion_config='{\"chunk_size\":1000}'" \
+    "-H 'Content-Type: multipart/form-data'"
+DOCUMENT_ID_FROM_FILE_WITH_CUSTOM_INGEST=${DOCUMENT_ID}
+
+echo "# 2. List Documents Tests"
+echo "=== Testing Document Listing ==="
+
+echo "# Test 2.1: List all documents"
+make_request "GET" "/documents" ""
+
+echo "# Test 2.2: List documents with pagination"
+make_request "GET" "/documents?offset=0&limit=10" ""
+
+echo "# Test 2.3: List specific documents by IDs"
+make_request "GET" "/documents?ids=${DOCUMENT_ID_FROM_CHUNKS},${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
+
+echo "# Test 2.4: List documents with summary embeddings"
+make_request "GET" "/documents?include_summary_embeddings=1" ""
+
+echo "# 3. Get Document Tests"
+echo "=== Testing Document Retrieval ==="
+
+echo "# Test 3.1: Get document by ID"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}" ""
+
+echo "# Test 3.2: Get non-existent document"
+make_request "GET" "/documents/00000000-0000-0000-0000-000000000000" ""
+
+echo "# 4. List Document Chunks Tests"
+echo "=== Testing Document Chunks ==="
+
+echo "# Test 4.1: List chunks with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?offset=0&limit=10" ""
+
+echo "# Test 4.2: List chunks with vectors"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/chunks?include_vectors=true" ""
+
+echo "# 5. Download Document Tests"
+echo "=== Testing Document Download ==="
+
+echo "# Test 5.1: Download document"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE}/download" ""
+
+echo "# 6. Delete Document Tests"
+echo "=== Testing Document Deletion ==="
+
+echo "# Test 6.1: Delete document by ID"
+make_request "DELETE" "/documents/${DOCUMENT_ID_FROM_RAW_TEXT}" ""
+
+echo "# Test 6.2: Delete documents by filter"
+make_request "DELETE" "/documents/by-filter" \
+    "-d '{\"document_type\":{\"\$eq\":\"txt\"}}'"
+
+echo "# 7. Document Collections Tests"
+echo "=== Testing Document Collections ==="
+
+echo "# Test 7.1: List document collections"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_CHUNKS}/collections?offset=0&limit=10" ""
+
+echo "# 8. Entity Extraction Tests"
+echo "=== Testing Entity Extraction ==="
+
+echo "# Test 8.1: Extract entities with default settings"
+make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
+    "-d '{\"run_type\":\"RUN\"}'"
+
+echo "# Test 8.2: Extract entities with custom settings"
+make_request "POST" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/extract" \
+    "-d '{\"run_type\":\"RUN\",\"settings\":{\"model\":\"gpt-4\"}}'"
+
+echo "# 9. Entity Retrieval Tests"
+echo "=== Testing Entity Retrieval ==="
+
+echo "# Test 9.1: Get entities with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?offset=0&limit=10" ""
+
+echo "# Test 9.2: Get entities with embeddings"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/entities?include_embeddings=true" ""
+
+echo "# 10. Relationship Tests"
+echo "=== Testing Relationships ==="
+
+echo "# Test 10.1: Get relationships with pagination"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?offset=0&limit=10" ""
+
+echo "# Test 10.2: Get relationships with filters"
+make_request "GET" "/documents/${DOCUMENT_ID_FROM_FILE_WITH_METADATA}/relationships?entity_names=person,company&relationship_types=works_for" ""
+
+echo "# 11. Document Search Tests"
+echo "=== Testing Document Search ==="
+
+echo "# Test 11.1: Basic search"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"basic\"}'"
+
+echo "# Test 11.2: Advanced search with custom settings"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"advanced\",\"search_settings\":{\"filters\":{\"document_type\":\"pdf\"}}}'"
+
+echo "# Test 11.3: Custom search with complex filters"
+make_request "POST" "/documents/search" \
+    "-d '{\"query\":\"test query\",\"search_mode\":\"custom\",\"search_settings\":{\"filters\":{\"created_at\":{\"\$gt\":\"2023-01-01\"}}}}'"

cleverbrag/tests/test_shared_functions.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+#
+# Generates random valid UUID
+#
+generate_uuid() {
+    local N B C='89ab'
+    local RES=''
+    for (( N=0; N < 16; ++N )) do
+        B=$(( $RANDOM % 256 ))
+
+        case $N in
+            6)
+                RES="${RES}$(printf '4%x' $(( B % 16 )))"
+                ;;
+            8)
+                RES="${RES}$(printf '%c%x' ${C:$RANDOM%${#C}:1} $(( B % 16 )))"
+                ;;
+            3 | 5 | 7 | 9)
+                RES="${RES}$(printf '%02x-' $B)"
+                ;;
+            *)
+                RES="${RES}$(printf '%02x' $B)"
+                ;;
+        esac
+    done
+    echo -n $RES
+}
+
+#
+# Extract document_id (if present) and HTTP response code from curl response passed as argument
+#
+extract_document_id() {
+  DOCUMENT_ID=$(echo $1 | grep document_id | tail -1 | sed -e 's/^.*"document_id": *"\([^"]*\)".*$/\1/')
+  HTTP_RETURN_CODE=$(echo echo $1 | tail -1 | sed -e 's/^.*\([0-9][0-9][0-9]\)$/\1/')
+}
+
+# Helper function to make curl requests
+make_request() {
+    local method=$1
+    local endpoint=$2
+    local data=$3
+    local headers=$4
+    if [ "xx${headers}" == "xx" ]; then
+      headers='-H "Content-Type: application/json"'
+    fi
+    # echo "Testing $method $endpoint"
+    # echo "Request data: $data"
+    # echo "Request headers: $headers"
+    CMD="curl -X $method \"$API_BASE$endpoint\" -s  -w \"\n%{http_code}\" -H \"Authorization: Bearer $AUTH_TOKEN\" $headers $data"
+    echo $CMD
+    RES=$(eval $CMD 2>/dev/null)
+    echo "Response: [${RES}]"
+    extract_document_id "$RES"
+    # echo "document_id --> ${DOCUMENT_ID}"
+    if [[ ${HTTP_RETURN_CODE} -lt 300 ]]; then
+      echo "SUCCESS. code = ${HTTP_RETURN_CODE}"
+    else
+      echo "ERROR. code = ${HTTP_RETURN_CODE}"
+    fi
+    echo -e "\n\n"
+    if [ "${EXECUTE_ALL}" != "-a" ]; then
+      echo "Press any key to continue..."
+      read -n 1 -s -r
+      echo -e "\n\n"
+    fi
+}
+
+#
+# Simple scanner of cmd line arguments, inferring meaning of the value
+# It recognized:
+#  a)  host URL (starts with 'http' prefix)
+#  b)  JWT token (starts with 'ey' prefix)
+#  c)  flag -a  to execute all tests w/o waiting for user pressing any key
+#
+while [ $# -gt 0 ]; do
+    echo "Argument: [${1}]"
+    if [[ "$1" == ey* ]]; then
+      AUTH_TOKEN=$1
+    fi
+    if [[ "$1" == http* ]]; then
+      HOST=$1
+    fi
+    if [[ "$1" == "-a" ]]; then
+      EXECUTE_ALL=$1
+    fi
+    shift  # Removes $1 and shifts remaining args left
+done
+
+
+if [ "xx${HOST}" == "xx" ]; then
+  HOST="http://localhost:7272"
+fi
+API_BASE="$HOST/v3"
+# Once integrated with CleverMicro, token can be obtained from CleverMicro
+# if [ "xx${AUTH_TOKEN}" == "xx" ]; then
+#  AUTH_TOKEN=$(curl $HOST/api/v0/login -s -d '{"username":"hugo@cleverthis.com","password":"tellmeall"}' -H "Content-Type: application/json" | sed -e "s/.*access_token...\(.*\)..$/\1/")
+# fi

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "amq_adapter"
-version = "0.2.32"
+version = "0.2.33"
 description = "The Python implementation of the AMQ Adapter for CleverMicro"
 readme = "README.md"
 authors = [

tests/adapter/test_backpressure_handler.py

@@ -284,7 +284,7 @@ class TestBackpressureHandler:
         assert handler.config == mock_config
         assert handler.swarm_service_id == "clever-amqp-service"
         assert handler.swarm_task_id == "python-adapter"
-        assert handler.parallel_workers == 1
+        assert handler.parallel_workers == mock_config.backpressure.threshold_threads
         assert handler.average_cpu_usage is None
 
     def test_increase_parallel_executions(self, mock_channel, mock_loop, mock_config):

tests/adapter/test_cleverthis_service_adapter.py

@@ -1,10 +1,13 @@
 import asyncio
 import unittest
+from enum import Enum
+from typing import Any, List, Optional
 from unittest.mock import AsyncMock, MagicMock, patch
 
 from aio_pika.abc import AbstractChannel, AbstractRobustConnection
 from aiohttp import ClientResponse
 from aiohttp.web_fileresponse import FileResponse
+from pydantic import BaseModel
 
 from amqp.adapter.cleverthis_service_adapter import (
     AMQMessage,
@@ -19,6 +22,26 @@ from amqp.model.model import DataResponse
 from amqp.router.router_base import AMQ_REPLY_TO_EXCHANGE
 
 
+class NestedModel(BaseModel):
+    id: int
+    name: str
+
+
+class ComplexModel(BaseModel):
+    nested: NestedModel
+    items: List[str]
+
+
+class TestEnum(Enum):
+    VALUE1 = "value1"
+    VALUE2 = "value2"
+
+
+class TestModel(BaseModel):
+    name: str
+    value: int
+
+
 class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase):
 
     async def test_create_reply_exchange_and_queue_success(self):
@@ -89,6 +112,7 @@ class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase):
             amq_configuration=self.mock_amq_config, routes=[], session=self.mock_session
         )
         self.adapter.service_host = "testhost"
+        self.adapter.authenticated_user_arg_name = "user"
 
     def test_init(self):
         self.assertEqual(self.adapter.service_host, "testhost")
@@ -282,6 +306,126 @@ class TestCleverThisServiceAdapter(unittest.IsolatedAsyncioTestCase):
         with self.assertRaises(NotImplementedError):
             await self.adapter.head(mock_message, None)
 
+    def test_basic_type_conversion(self):
+        async def test_method(x: int, y: float, z: bool, text: str):
+            pass
+
+        args = {"x": "123", "y": "45.67", "z": "true", "text": "hello"}
+
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["x"], 123)
+        self.assertEqual(result["y"], 45.67)
+        self.assertEqual(result["z"], True)
+        self.assertEqual(result["text"], "hello")
+
+    def test_enum_conversion(self):
+        async def test_method(status: TestEnum):
+            pass
+
+        args = {"status": "value1"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["status"], TestEnum.VALUE1)
+
+    def test_optional_parameter(self):
+        async def test_method(required: str, optional: Optional[str] = None):
+            pass
+
+        args = {"required": "test"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["required"], "test")
+        self.assertIsNone(result["optional"])
+
+    def test_list_parameter(self):
+        async def test_method(items: List[str]):
+            pass
+
+        args = {"items": "item1,item2,item3"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["items"], ["item1", "item2", "item3"])
+
+    def test_pydantic_model_parameter(self):
+        async def test_method(model: TestModel):
+            pass
+
+        args = {"model": '{"name": "test", "value": 42}'}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertIsInstance(result["model"], TestModel)
+        self.assertEqual(result["model"].name, "test")
+        self.assertEqual(result["model"].value, 42)
+
+    def test_authenticated_user_parameter(self):
+        async def test_method(user: Any):
+            pass
+
+        user_obj = MagicMock()
+        args = {"user": user_obj}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["user"], user_obj)
+
+    def test_missing_required_parameter(self):
+        async def test_method(required: str):
+            pass
+
+        args = {}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertNotIn("required", result)
+
+    def test_invalid_enum_value(self):
+        async def test_method(status: TestEnum):
+            pass
+
+        args = {"status": "invalid_value"}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["status"], "invalid_value")
+
+    def test_invalid_type_conversion(self):
+        async def test_method(x: int):
+            pass
+
+        args = {"x": "not_a_number"}
+        result = {}
+        try:
+            result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+        except Exception as e:
+            self.assertTrue(isinstance(e, TypeError))
+
+        self.assertEqual(len(result), 0)
+
+    def test_complex_nested_model(self):
+
+        async def test_method(model: ComplexModel):
+            pass
+
+        args = {"model": {"nested": {"id": 1, "name": "test"}, "items": ["item1", "item2"]}}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertIsInstance(result["model"], ComplexModel)
+        self.assertEqual(result["model"].nested.id, 1)
+        self.assertEqual(result["model"].nested.name, "test")
+        self.assertEqual(result["model"].items, ["item1", "item2"])
+
+    def test_default_factory_parameter(self):
+        class DefaultFactory:
+            @staticmethod
+            def default_factory():
+                return "default_value"
+
+        async def test_method(param: str = DefaultFactory()):
+            pass
+
+        args = {}
+        result = self.adapter._verify_and_instantiate_arguments(args, test_method)
+
+        self.assertEqual(result["param"], "default_value")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/adapter/test_pydantic_serializer.py

@@ -1,4 +1,5 @@
-from datetime import datetime
+import json
+from datetime import date, datetime
 from unittest import TestCase
 from uuid import UUID
 
@@ -7,6 +8,7 @@ from pydantic import BaseModel
 from amqp.adapter.pydantic_serializer import (
     deserialize_object,
     parse_url_query,
+    python_type_to_json,
     serialize_object,
 )
 
@@ -130,3 +132,95 @@ class PydanticSerializerTest(TestCase):
         )
         self.assertEqual(path, "/api/v0/unstructured/863e4d9f-4612-4173-8d0a-3de34cc7a741")
         self.assertEqual(args, {})
+
+
+class TestPythonTypeToJson:
+    """
+    Test cases for the python_type_to_json function.
+    """
+
+    def test_basic_types(self):
+        """Test serialization of basic Python types."""
+        # Test dictionary
+        data = {"key": "value", "number": 42}
+        result = python_type_to_json(data)
+        assert json.loads(result) == data
+
+        # Test list
+        data = [1, 2, 3, "four"]
+        result = python_type_to_json(data)
+        assert json.loads(result) == data
+
+        # Test tuple
+        data = (1, 2, 3)
+        result = python_type_to_json(data)
+        assert json.loads(result) == [1, 2, 3]  # Tuples are converted to lists
+
+        # Test set
+        data = {1, 2, 3}
+        result = python_type_to_json(data)
+        assert set(json.loads(result)) == data  # Sets are converted to lists
+
+    def test_datetime_handling(self):
+        """Test serialization of datetime objects."""
+        # Test datetime
+        test_datetime = datetime(2024, 1, 1, 12, 0, 0)
+        data = {"timestamp": test_datetime}
+        result = python_type_to_json(data)
+        assert json.loads(result) == {"timestamp": test_datetime.isoformat()}
+
+        # Test date
+        test_date = date(2024, 1, 1)
+        data = {"date": test_date}
+        result = python_type_to_json(data)
+        assert json.loads(result) == {"date": test_date.isoformat()}
+
+    def test_nested_structures(self):
+        """Test serialization of nested data structures."""
+        data = {
+            "list_of_datetimes": [datetime(2024, 1, 1), datetime(2024, 1, 2)],
+            "dict_with_date": {"date": date(2024, 1, 1)},
+            "mixed": {"set": {1, 2, 3}, "tuple": (1, 2, 3), "datetime": datetime(2024, 1, 1)},
+        }
+        result = python_type_to_json(data)
+        parsed = json.loads(result)
+        assert parsed["list_of_datetimes"] == [d.isoformat() for d in data["list_of_datetimes"]]
+        assert parsed["dict_with_date"]["date"] == data["dict_with_date"]["date"].isoformat()
+        assert set(parsed["mixed"]["set"]) == data["mixed"]["set"]
+        assert parsed["mixed"]["tuple"] == list(data["mixed"]["tuple"])
+        assert parsed["mixed"]["datetime"] == data["mixed"]["datetime"].isoformat()
+
+    def test_custom_object_serialization(self):
+        """Test serialization of custom objects with __dict__ attribute."""
+
+        class TestObject:
+            def __init__(self, value):
+                self.value = value
+                self.timestamp = datetime(2024, 1, 1)
+
+        obj = TestObject("test")
+        data = {"obj": obj}
+        result = python_type_to_json(data)
+        parsed = json.loads(result)
+        assert parsed["obj"]["value"] == "test"
+        assert parsed["obj"]["timestamp"] == obj.timestamp.isoformat()
+
+    def test_empty_structures(self):
+        """Test serialization of empty data structures."""
+        # Empty dict
+        assert python_type_to_json({}) == "{}"
+
+        # Empty list
+        assert python_type_to_json([]) == "[]"
+
+        # Empty set
+        assert python_type_to_json(set()) == "[]"
+
+        # Empty tuple
+        assert python_type_to_json(()) == "[]"
+
+    def test_unicode_handling(self):
+        """Test handling of Unicode characters."""
+        data = {"text": "Hello, 世界!"}
+        result = python_type_to_json(data)
+        assert json.loads(result) == data  # ensure_ascii=False should preserve Unicode

tests/adapter/test_service_message_factory.py

@@ -1,9 +1,9 @@
 from unittest import TestCase
 
 from amqp.adapter.service_message_factory import (
-    ServiceMessageFactory,
-    RS,
     INVALID_MESSAGE,
+    RS,
+    ServiceMessageFactory,
 )
 from amqp.model.model import ServiceMessage
 from amqp.model.service_message_type import ServiceMessageType
@@ -13,15 +13,11 @@ class ServiceMessageTest(TestCase):
 
     def test_to_string(self):
         _f: ServiceMessageFactory = ServiceMessageFactory("bumble-bee")
-        message: ServiceMessage = _f.of(
-            ServiceMessageType.ROUTING_DATA, "Duck", "Donald"
-        )
+        message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         message_str = _f.to_string(message)
         self.assertTrue("|type=01|" in message_str)
         self.assertTrue("|body=Duck" in message_str)
-        message: ServiceMessage = _f.of(
-            ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald"
-        )
+        message: ServiceMessage = _f.of(ServiceMessageType.ROUTING_DATA_REPLACE, "Duck", "Donald")
         message_str = _f.to_string(message)
         self.assertTrue("|type=03|" in message_str)
         self.assertTrue("|body=Duck" in message_str)
@@ -57,3 +53,4 @@ class ServiceMessageTest(TestCase):
         message = _f.of(ServiceMessageType.ROUTING_DATA, "Duck", "Donald")
         serialized = _f.serialize(message)
         assert RS + "0" + str(ServiceMessageType.ROUTING_DATA.value)
+        assert len(serialized) > 16

tests/adapter/test_type_descriptor.py

@@ -0,0 +1,211 @@
+import json
+import unittest
+
+from pydantic import BaseModel
+
+from amqp.adapter.type_descriptor import TypeDescriptor, _instantiate
+
+
+class TestClass(BaseModel):
+    value: str
+    extra: str
+
+
+class TestTypeDescriptor(unittest.TestCase):
+    def test_annotated_with_json(self):
+        type_str = (
+            "typing.Optional[typing.Annotated[core.base.providers.ingestion.IngestionConfig, Json]]"
+        )
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "core.base.providers.ingestion.IngestionConfig")
+        self.assertTrue(td.is_json)
+
+    def test_annotated_without_json(self):
+        type_str = "typing.Optional[typing.Annotated[str, SomethingElse]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "str")
+        self.assertFalse(td.is_json)
+
+    def test_annotated_multiple_args(self):
+        type_str = "typing.Optional[typing.Annotated[list[int], Json, more_args]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "list[int]")
+        self.assertTrue(td.is_json)
+
+    def test_simple_optional(self):
+        type_str = "typing.Optional[uuid.UUID]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(td.type, "uuid.UUID")
+        self.assertFalse(td.is_json)
+
+    def test_whitespace_variations(self):
+        test_cases = [
+            ("typing.Optional[typing.Annotated[  str  ,  Json  ]]", "str", True),
+            ("typing.Optional[  typing.Annotated[dict,Json]]", "dict", True),
+            ("typing.Optional[  int  ]", "int", False),
+        ]
+
+        for type_str, expected_type, expected_json in test_cases:
+            with self.subTest(type_str=type_str):
+                td = TypeDescriptor(type_str, {})
+                self.assertEqual(td.type, expected_type)
+                self.assertEqual(td.is_json, expected_json)
+
+    def test_invalid_input(self):
+        test_cases = [
+            "typing.Optional[missing_bracket",
+            "typing.Optional[]",
+            "[without.optional]",
+        ]
+
+        for type_str in test_cases:
+            with self.subTest(type_str=type_str):
+                td = TypeDescriptor(type_str, {})
+                self.assertEqual(td.type, "")
+                self.assertFalse(td.is_json)
+
+    def test_repr(self):
+        type_str = "typing.Optional[typing.Annotated[str, Json]]"
+        td = TypeDescriptor(type_str, {})
+
+        self.assertEqual(repr(td), "TypeDescriptor(type='str', is_json=True)")
+
+    def test_instantiate_basic_types(self):
+        test_cases = [
+            ("str", "hello", "hello"),
+            ("int", "42", 42),
+            ("float", "3.14", 3.14),
+            ("bool", "true", True),
+        ]
+
+        for type_str, input_value, expected in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value):
+                result = _instantiate(type_str, input_value)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_list_types(self):
+        test_cases = [
+            ("typing.List[str]", "a,b,c", ["a", "b", "c"]),
+            ("typing.List[int]", "1,2,3", [1, 2, 3]),
+            ("typing.List[str]", json.dumps(["a", "b", "c"]), ["a", "b", "c"], True),
+            ("typing.List[int]", json.dumps([1, 2, 3]), [1, 2, 3], True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_dict_types(self):
+        test_cases = [
+            ("typing.Dict", {"a": 1, "b": 2}, {"a": 1, "b": 2}),
+            ("typing.Dict", json.dumps({"a": 1, "b": 2}), {"a": 1, "b": 2}, True),
+            ("typing.Mapping", {"a": 1, "b": 2}, {"a": 1, "b": 2}),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_set_types(self):
+        test_cases = [
+            ("typing.Set[str]", "a,b,c", {"a", "b", "c"}),
+            ("typing.Set[int]", "1,2,3", {1, 2, 3}),
+            ("typing.Set[str]", json.dumps(["a", "b", "c"]), {"a", "b", "c"}, True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_tuple_types(self):
+        test_cases = [
+            ("typing.Tuple[str, int]", ["hello", 42], ("hello", 42)),
+            ("typing.Tuple[str, int]", json.dumps(["hello", 42]), ("hello", 42), True),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_optional_types(self):
+        test_cases = [
+            ("typing.Optional[str]", "hello", "hello"),
+            ("typing.Optional[typing.List[str]]", "a,b,c", ["a", "b", "c"]),
+            (
+                "typing.Optional[typing.List[str]]",
+                json.dumps(["a", "b", "c"]),
+                ["a", "b", "c"],
+                True,
+            ),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_nested_types(self):
+        test_cases = [
+            (
+                "typing.List[typing.Dict]",
+                json.dumps([{"a": 1}, {"b": 2}]),
+                [{"a": 1}, {"b": 2}],
+                True,
+            ),
+            (
+                "typing.Optional[typing.List[typing.Dict]]",
+                json.dumps([{"a": 1}, {"b": 2}]),
+                [{"a": 1}, {"b": 2}],
+                True,
+            ),
+        ]
+
+        for type_str, input_value, expected, *args in test_cases:
+            is_json = args[0] if args else False
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                result = _instantiate(type_str, input_value, is_json=is_json)
+                self.assertEqual(result, expected)
+
+    def test_instantiate_error_handling(self):
+        test_cases = [
+            ("typing.List[str]", "invalid json", True, json.JSONDecodeError),
+            ("typing.NonExistentType", "value", False, ImportError),
+        ]
+
+        for type_str, input_value, is_json, expected_exception in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value, is_json=is_json):
+                with self.assertRaises(expected_exception):
+                    _instantiate(type_str, input_value, is_json=is_json)
+
+    def test_instantiate_with_extra_init_data(self):
+
+        test_cases = [
+            (
+                "tests.adapter.test_type_descriptor.TestClass",
+                {"value": "test"},
+                {"extra": "extra_value"},
+                lambda x: x.value == "test" and x.extra == "extra_value",
+            ),
+        ]
+
+        for type_str, input_value, extra_init_data, validator in test_cases:
+            with self.subTest(type_str=type_str, input_value=input_value):
+                result = _instantiate(type_str, input_value, extra_init_data=extra_init_data)
+                self.assertTrue(validator(result))
+
+
+if __name__ == "__main__":
+    unittest.main()