freemo
7c4663b8ee
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / quality (pull_request) Successful in 18s
CI / build (pull_request) Successful in 19s
CI / typecheck (pull_request) Successful in 38s
CI / security (pull_request) Successful in 46s
CI / integration_tests (pull_request) Successful in 2m52s
CI / unit_tests (pull_request) Successful in 12m35s
CI / docker (pull_request) Successful in 38s
CI / benchmark-regression (pull_request) Successful in 20m57s
CI / coverage (pull_request) Successful in 47m23s
CI / lint (push) Successful in 12s
CI / build (push) Successful in 14s
CI / quality (push) Successful in 17s
CI / typecheck (push) Successful in 32s
CI / benchmark-regression (push) Has been skipped
CI / security (push) Successful in 40s
CI / integration_tests (push) Successful in 2m52s
CI / benchmark-publish (push) Successful in 12m29s
CI / unit_tests (push) Successful in 12m31s
CI / docker (push) Successful in 38s
CI / coverage (push) Successful in 47m17s
Implement the complete configuration system with multi-level resolution chain, typed key registry, and CLI integration per specification. ConfigService changes: - Expand _build_catalog() to register all 102 spec-aligned config keys across 8 groups: core (14), server (4), actor (5), plan (8), sandbox (5), index (12), context (43), provider (11) - Each key carries exact dotted-dash name, Python type, default value, explicit env var name per spec, project-scopability flag, and description - Fix _env_name() to convert dots and dashes to underscores - Provider keys use standard env var names (e.g., OPENAI_API_KEY) CLI commands rewiring: - Rewrite config set/get/list to use ConfigService instead of Settings - Add --verbose flag to config get showing full 5-level resolution chain - Add --project flag to config set/get/list for project-scoped overrides - Support both glob and regex patterns in config list - Validate keys against ConfigService registry with actionable errors - Retain backward-compatible helper functions delegating to ConfigService Documentation: - Add docs/reference/config_resolution.md covering resolution chain, all 102 config keys, CLI commands, TOML format, and provider credentials Testing: - Update all 4 Behave feature files and step definitions to use new spec-aligned key names, env vars, and defaults (119 scenarios passing) - Add robot/config_resolution.robot with 10 integration test cases - Add benchmarks/config_resolution_bench.py with 8 time + 2 memory suites ISSUES CLOSED: #258
115 lines
3.7 KiB
Python
115 lines
3.7 KiB
Python
"""ASV benchmarks for Config CLI command throughput.
|
|
|
|
Measures the performance of:
|
|
- Config list (all settings)
|
|
- Config list with regex filter
|
|
- Config get (single key resolution)
|
|
- Config set (write + read roundtrip)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
# Ensure the local *source* tree is importable even when ASV has an
|
|
# older build of the package installed.
|
|
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
|
if _SRC not in sys.path:
|
|
sys.path.insert(0, _SRC)
|
|
|
|
import cleveragents # noqa: E402
|
|
|
|
importlib.reload(cleveragents)
|
|
|
|
from typer.testing import CliRunner # noqa: E402
|
|
|
|
from cleveragents.cli.commands import config as config_mod # noqa: E402
|
|
from cleveragents.cli.commands.config import app as config_app # noqa: E402
|
|
|
|
_runner = CliRunner()
|
|
|
|
|
|
class ConfigCLIListSuite:
|
|
"""Benchmark config list throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._tmpdir = Path(tempfile.mkdtemp())
|
|
self._tmppath = self._tmpdir / "config.toml"
|
|
self._dir_patch = patch.object(config_mod, "_CONFIG_DIR", self._tmpdir)
|
|
self._path_patch = patch.object(config_mod, "_CONFIG_PATH", self._tmppath)
|
|
self._dir_patch.start()
|
|
self._path_patch.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._dir_patch.stop()
|
|
self._path_patch.stop()
|
|
shutil.rmtree(str(self._tmpdir), ignore_errors=True)
|
|
|
|
def time_list_all(self) -> None:
|
|
"""Benchmark listing all config settings."""
|
|
_runner.invoke(config_app, ["list"])
|
|
|
|
def time_list_with_filter(self) -> None:
|
|
"""Benchmark listing with regex filter."""
|
|
_runner.invoke(config_app, ["list", "log.*"])
|
|
|
|
def time_list_json(self) -> None:
|
|
"""Benchmark listing in JSON format."""
|
|
_runner.invoke(config_app, ["list", "--format", "json"])
|
|
|
|
|
|
class ConfigCLIGetSuite:
|
|
"""Benchmark config get throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._tmpdir = Path(tempfile.mkdtemp())
|
|
self._tmppath = self._tmpdir / "config.toml"
|
|
self._dir_patch = patch.object(config_mod, "_CONFIG_DIR", self._tmpdir)
|
|
self._path_patch = patch.object(config_mod, "_CONFIG_PATH", self._tmppath)
|
|
self._dir_patch.start()
|
|
self._path_patch.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._dir_patch.stop()
|
|
self._path_patch.stop()
|
|
shutil.rmtree(str(self._tmpdir), ignore_errors=True)
|
|
|
|
def time_get_key(self) -> None:
|
|
"""Benchmark getting a single config key."""
|
|
_runner.invoke(config_app, ["get", "core.log.level"])
|
|
|
|
def time_get_key_json(self) -> None:
|
|
"""Benchmark getting a key in JSON format."""
|
|
_runner.invoke(config_app, ["get", "core.log.level", "--format", "json"])
|
|
|
|
|
|
class ConfigCLISetSuite:
|
|
"""Benchmark config set throughput."""
|
|
|
|
def setup(self) -> None:
|
|
self._tmpdir = Path(tempfile.mkdtemp())
|
|
self._tmppath = self._tmpdir / "config.toml"
|
|
self._dir_patch = patch.object(config_mod, "_CONFIG_DIR", self._tmpdir)
|
|
self._path_patch = patch.object(config_mod, "_CONFIG_PATH", self._tmppath)
|
|
self._dir_patch.start()
|
|
self._path_patch.start()
|
|
|
|
def teardown(self) -> None:
|
|
self._dir_patch.stop()
|
|
self._path_patch.stop()
|
|
shutil.rmtree(str(self._tmpdir), ignore_errors=True)
|
|
|
|
def time_set_key(self) -> None:
|
|
"""Benchmark setting a config value."""
|
|
_runner.invoke(config_app, ["set", "core.log.level", "DEBUG"])
|
|
|
|
def time_set_get_roundtrip(self) -> None:
|
|
"""Benchmark set followed by get."""
|
|
_runner.invoke(config_app, ["set", "core.log.level", "DEBUG"])
|
|
_runner.invoke(config_app, ["get", "core.log.level"])
|