freemo
7c4663b8ee
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 18s
CI / build (pull_request) Successful in 19s
CI / typecheck (pull_request) Successful in 38s
CI / security (pull_request) Successful in 46s
CI / integration_tests (pull_request) Successful in 2m52s
CI / unit_tests (pull_request) Successful in 12m35s
CI / docker (pull_request) Successful in 38s
CI / benchmark-regression (pull_request) Successful in 20m57s
CI / coverage (pull_request) Successful in 47m23s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 14s
CI / quality (push) Successful in 17s
CI / typecheck (push) Successful in 32s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 40s
CI / integration_tests (push) Successful in 2m52s
CI / benchmark-publish (push) Successful in 12m29s
CI / unit_tests (push) Successful in 12m31s
CI / docker (push) Successful in 38s
CI / coverage (push) Successful in 47m17s
Implement the complete configuration system with multi-level resolution chain, typed key registry, and CLI integration per specification. ConfigService changes: - Expand _build_catalog() to register all 102 spec-aligned config keys across 8 groups: core (14), server (4), actor (5), plan (8), sandbox (5), index (12), context (43), provider (11) - Each key carries exact dotted-dash name, Python type, default value, explicit env var name per spec, project-scopability flag, and description - Fix _env_name() to convert dots and dashes to underscores - Provider keys use standard env var names (e.g., OPENAI_API_KEY) CLI commands rewiring: - Rewrite config set/get/list to use ConfigService instead of Settings - Add --verbose flag to config get showing full 5-level resolution chain - Add --project flag to config set/get/list for project-scoped overrides - Support both glob and regex patterns in config list - Validate keys against ConfigService registry with actionable errors - Retain backward-compatible helper functions delegating to ConfigService Documentation: - Add docs/reference/config_resolution.md covering resolution chain, all 102 config keys, CLI commands, TOML format, and provider credentials Testing: - Update all 4 Behave feature files and step definitions to use new spec-aligned key names, env vars, and defaults (119 scenarios passing) - Add robot/config_resolution.robot with 10 integration test cases - Add benchmarks/config_resolution_bench.py with 8 time + 2 memory suites ISSUES CLOSED: #258
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""ASV benchmarks for Config Service resolution chain performance.
|
|
|
|
Measures the performance of:
|
|
- Single key resolution (default, env var, verbose chain)
|
|
- Full registry resolution (resolve_all)
|
|
- Key validation and type coercion
|
|
- Registry lookup and enumeration
|
|
- Memory consumption for bulk operations
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from cleveragents.application.services.config_service import ( # noqa: E402
|
|
ConfigService,
|
|
)
|
|
|
|
|
|
class ConfigResolutionTimeSuite:
|
|
"""Benchmark timing for config resolution, validation, and registry ops."""
|
|
|
|
def setup(self) -> None:
|
|
self._tmpdir = Path(tempfile.mkdtemp())
|
|
self._service = ConfigService(
|
|
config_dir=self._tmpdir,
|
|
config_path=self._tmpdir / "config.toml",
|
|
)
|
|
|
|
def teardown(self) -> None:
|
|
os.environ.pop("CLEVERAGENTS_LOG_LEVEL", None)
|
|
shutil.rmtree(str(self._tmpdir), ignore_errors=True)
|
|
|
|
def time_resolve_single_key(self) -> None:
|
|
"""Resolve a single key (core.log.level) with no overrides."""
|
|
self._service.resolve("core.log.level")
|
|
|
|
def time_resolve_all_keys(self) -> None:
|
|
"""Resolve all 103 registered keys via resolve_all()."""
|
|
self._service.resolve_all()
|
|
|
|
def time_resolve_with_env_var(self) -> None:
|
|
"""Resolve with an env var set."""
|
|
os.environ["CLEVERAGENTS_LOG_LEVEL"] = "DEBUG"
|
|
self._service.resolve("core.log.level")
|
|
|
|
def time_resolve_verbose(self) -> None:
|
|
"""Resolve with verbose=True (builds resolution chain)."""
|
|
self._service.resolve("core.log.level", verbose=True)
|
|
|
|
def time_validate_key(self) -> None:
|
|
"""Call validate_key() for a known key."""
|
|
ConfigService.validate_key("core.log.level")
|
|
|
|
def time_validate_type_coercion(self) -> None:
|
|
"""Call validate_type() for str->int coercion."""
|
|
ConfigService.validate_type("core.log.retention-days", "90")
|
|
|
|
def time_registry_lookup(self) -> None:
|
|
"""Look up an entry via get_entry()."""
|
|
ConfigService.get_entry("core.log.level")
|
|
|
|
def time_registered_keys_sorted(self) -> None:
|
|
"""Get sorted list of all registered keys."""
|
|
ConfigService.registered_keys()
|
|
|
|
|
|
class ConfigResolutionMemSuite:
|
|
"""Benchmark memory consumption for bulk config operations."""
|
|
|
|
def setup(self) -> None:
|
|
self._tmpdir = Path(tempfile.mkdtemp())
|
|
self._service = ConfigService(
|
|
config_dir=self._tmpdir,
|
|
config_path=self._tmpdir / "config.toml",
|
|
)
|
|
|
|
def teardown(self) -> None:
|
|
shutil.rmtree(str(self._tmpdir), ignore_errors=True)
|
|
|
|
def mem_resolve_all(self) -> dict:
|
|
"""Memory for resolving all keys."""
|
|
return self._service.resolve_all()
|
|
|
|
def mem_registry_copy(self) -> dict:
|
|
"""Memory for .registry() (dict copy)."""
|
|
return ConfigService.registry()
|