forked from HAL9000/cleveragents-core
freemo
c65e8a5285
Implement cloud resource types (aws, gcp, azure) with credential fields, region/tenant metadata, and stubbed sandbox strategies. Credential resolution uses environment variables and profile names with no secrets logged. Key changes: - Add CloudResourceHandler with aws/gcp/azure type definitions - Add credential resolution from env vars and profile names - Add stubbed sandbox strategies (validate config, raise NotImplementedError) - Register cloud types in bootstrap_builtin_types - Credential masking via existing redaction patterns - Add Behave BDD tests, Robot integration tests, ASV benchmarks ISSUES CLOSED: #343
107 lines
3.4 KiB
Python
107 lines
3.4 KiB
Python
"""ASV benchmarks for actor list on fresh project.
|
|
|
|
Measures the cost of listing actors via the ``ActorRegistry`` when
|
|
no providers are configured and when a provider with a multi-slash
|
|
default model is present.
|
|
|
|
Targets bug #592 — ``_actor_name()`` now sanitises provider and model
|
|
names by replacing ``/`` with ``-`` and lowercasing, so multi-slash
|
|
provider models no longer trigger ``ValidationError``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Ensure the features directory is importable so we can reuse shared
|
|
# mock stubs instead of duplicating them locally.
|
|
_FEATURES = str(Path(__file__).resolve().parents[1] / "features")
|
|
if _FEATURES not in sys.path:
|
|
sys.path.insert(0, _FEATURES)
|
|
|
|
# Force-reload so ASV picks up the source tree version.
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from mocks.fake_provider import FakeProviderInfo, FakeProviderRegistry # noqa: E402
|
|
|
|
from cleveragents.actor.registry import ActorRegistry # noqa: E402
|
|
from cleveragents.core.exceptions import ValidationError # noqa: E402
|
|
from cleveragents.domain.models.core.actor import Actor # noqa: E402
|
|
|
|
|
|
def _make_registry(
|
|
providers: list[FakeProviderInfo] | None = None,
|
|
) -> ActorRegistry:
|
|
"""Build an ``ActorRegistry`` with mocked dependencies."""
|
|
prov_reg = FakeProviderRegistry(providers=providers or [])
|
|
mock_service = MagicMock()
|
|
mock_service.list_actors.return_value = []
|
|
mock_service.get_default_actor.return_value = None
|
|
mock_service.upsert_actor.side_effect = lambda **kw: Actor(
|
|
name=kw.get("name", "mock/actor"),
|
|
provider=kw.get("provider", "mock"),
|
|
model=kw.get("model", "actor"),
|
|
config_blob={},
|
|
config_hash=Actor.compute_hash({}),
|
|
)
|
|
mock_settings = MagicMock()
|
|
mock_settings.resolve_provider_defaults.return_value = MagicMock(
|
|
provider=None, model=None
|
|
)
|
|
return ActorRegistry(
|
|
actor_service=mock_service,
|
|
provider_registry=prov_reg,
|
|
settings=mock_settings,
|
|
)
|
|
|
|
|
|
class ActorListEmptySuite:
|
|
"""Benchmark actor list with zero and multi-slash providers."""
|
|
|
|
timeout = 60
|
|
|
|
def setup(self) -> None:
|
|
self._empty_registry = _make_registry()
|
|
self._slash_registry = _make_registry(
|
|
providers=[
|
|
FakeProviderInfo(
|
|
name="Openrouter",
|
|
default_model="anthropic/claude-sonnet-4-20250514",
|
|
),
|
|
]
|
|
)
|
|
|
|
def time_list_empty(self) -> None:
|
|
"""List actors with zero configured providers."""
|
|
self._empty_registry.list_actors()
|
|
|
|
def time_list_multi_slash_provider(self) -> None:
|
|
"""List actors when a multi-slash provider is configured."""
|
|
self._slash_registry.list_actors()
|
|
|
|
def track_multi_slash_succeeds(self) -> int:
|
|
"""Track whether listing with a multi-slash provider succeeds.
|
|
|
|
Returns 1 when list_actors() completes without error; 0 if
|
|
ValidationError is raised.
|
|
"""
|
|
try:
|
|
self._slash_registry.list_actors()
|
|
return 1
|
|
except ValidationError:
|
|
return 0
|
|
|
|
|
|
ActorListEmptySuite.track_multi_slash_succeeds.unit = "success"
|