freemo
92c83ecc7e
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 16s
CI / build (pull_request) Successful in 17s
CI / quality (pull_request) Successful in 19s
CI / typecheck (pull_request) Successful in 33s
CI / security (pull_request) Successful in 35s
CI / integration_tests (pull_request) Successful in 2m41s
CI / unit_tests (pull_request) Successful in 6m18s
CI / docker (pull_request) Successful in 1m2s
CI / benchmark-regression (pull_request) Successful in 15m41s
CI / coverage (pull_request) Failing after 20m52s
112 lines
3.2 KiB
Python
112 lines
3.2 KiB
Python
"""ASV benchmarks for actor registry list performance.
|
|
|
|
Measures the performance of:
|
|
- Actor creation with YAML text and metadata
|
|
- Actor listing with and without namespace filters
|
|
- Schema version tracking overhead
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import sys
|
|
import typing
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
# Force-reload the top-level package
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.domain.models.core.actor import Actor # noqa: E402
|
|
|
|
|
|
def _make_actor(
|
|
name: str,
|
|
*,
|
|
yaml_text: str | None = None,
|
|
schema_version: str = "1.0",
|
|
compiled_metadata: dict | None = None,
|
|
) -> Actor:
|
|
"""Create a test actor with registry-aligned fields."""
|
|
blob: dict = {"provider": "openai", "model": "gpt-4"}
|
|
return Actor(
|
|
id=None,
|
|
name=name,
|
|
provider="openai",
|
|
model="gpt-4",
|
|
config_blob=blob,
|
|
config_hash=Actor.compute_hash(blob),
|
|
yaml_text=yaml_text,
|
|
schema_version=schema_version,
|
|
compiled_metadata=compiled_metadata,
|
|
)
|
|
|
|
|
|
class TimeActorCreation:
|
|
"""Benchmark actor creation with YAML text and metadata."""
|
|
|
|
_YAML = "name: bench/actor\nprovider: openai\nmodel: gpt-4\n"
|
|
_META: typing.ClassVar[dict] = {"graph_nodes": ["a", "b"], "tool_count": 5}
|
|
|
|
def time_create_minimal(self) -> None:
|
|
_make_actor("bench/minimal")
|
|
|
|
def time_create_with_yaml(self) -> None:
|
|
_make_actor("bench/yaml", yaml_text=self._YAML)
|
|
|
|
def time_create_with_metadata(self) -> None:
|
|
_make_actor("bench/meta", yaml_text=self._YAML, compiled_metadata=self._META)
|
|
|
|
def time_create_with_schema_version(self) -> None:
|
|
_make_actor("bench/versioned", schema_version="2.0")
|
|
|
|
|
|
class TimeActorListing:
|
|
"""Benchmark actor listing and namespace filtering."""
|
|
|
|
def setup(self) -> None:
|
|
self.actors: list[Actor] = []
|
|
for i in range(100):
|
|
ns = "local" if i % 2 == 0 else "remote"
|
|
self.actors.append(
|
|
_make_actor(
|
|
f"{ns}/actor-{i}",
|
|
yaml_text=f"name: {ns}/actor-{i}\nprovider: openai\nmodel: gpt-4\n",
|
|
schema_version="1.0",
|
|
)
|
|
)
|
|
|
|
def time_list_all(self) -> None:
|
|
_ = list(self.actors)
|
|
|
|
def time_list_with_namespace_filter(self) -> None:
|
|
prefix = "local/"
|
|
_ = [a for a in self.actors if a.name.startswith(prefix)]
|
|
|
|
def time_list_schema_version_filter(self) -> None:
|
|
_ = [a for a in self.actors if a.schema_version == "1.0"]
|
|
|
|
|
|
class TimeComputeHash:
|
|
"""Benchmark config hash computation."""
|
|
|
|
_SMALL_BLOB: typing.ClassVar[dict] = {"provider": "openai", "model": "gpt-4"}
|
|
_LARGE_BLOB: typing.ClassVar[dict] = {
|
|
"provider": "openai",
|
|
"model": "gpt-4",
|
|
"options": {f"key_{i}": f"value_{i}" for i in range(100)},
|
|
"graph_descriptor": {"nodes": list(range(50))},
|
|
}
|
|
|
|
def time_hash_small_blob(self) -> None:
|
|
Actor.compute_hash(self._SMALL_BLOB)
|
|
|
|
def time_hash_large_blob(self) -> None:
|
|
Actor.compute_hash(self._LARGE_BLOB)
|