a074b4846f
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 19s
CI / quality (pull_request) Successful in 28s
CI / security (pull_request) Successful in 50s
CI / typecheck (pull_request) Successful in 56s
CI / integration_tests (pull_request) Successful in 4m51s
CI / unit_tests (pull_request) Successful in 19m29s
CI / docker (pull_request) Successful in 39s
CI / benchmark-regression (pull_request) Successful in 26m10s
CI / coverage (pull_request) Successful in 47m42s
CI / lint (push) Successful in 13s
CI / quality (push) Successful in 17s
CI / build (push) Successful in 23s
CI / typecheck (push) Successful in 30s
CI / security (push) Successful in 30s
CI / benchmark-regression (push) Has been skipped
CI / integration_tests (push) Successful in 2m52s
CI / unit_tests (push) Successful in 10m8s
CI / docker (push) Successful in 1m19s
CI / benchmark-publish (push) Successful in 11m54s
CI / coverage (push) Failing after 39m51s
Remove FakeListLLM as a silent fallback in agent graph constructors (plan_generation.py, context_analysis.py, auto_debug.py). All three now raise ValueError when llm=None, making missing-provider errors explicit. Add Settings.mock_providers flag and validate_provider_availability() method. Update container.get_ai_provider() to check Settings.mock_providers first, with env-var fallback for backward compatibility. Add resolve_provider_by_name() helper to the provider registry and export it from cleveragents.providers. Add structlog trace logging to ProviderRegistry.get_default_provider_type() to record selection reasoning. Update all existing behave step files, robot tests, and benchmarks that relied on the implicit FakeListLLM default to pass an explicit LLM instance instead. Add new BDD tests (features/provider_fixes.feature with 17 scenarios), Robot Framework integration tests (robot/provider_detection_smoke.robot), and ASV benchmarks (benchmarks/provider_selection_bench.py). ISSUES CLOSED: #323
392 lines
15 KiB
Python
392 lines
15 KiB
Python
"""Step definitions for container_coverage_r2.feature.
|
|
|
|
Covers remaining uncovered lines and partial branches in
|
|
``cleveragents.application.container``:
|
|
|
|
- Lines 66-67, 69: get_ai_provider non-mock paths
|
|
- Lines 125-130: _build_project_resource_link_repo
|
|
- Partial branches at lines 51, 57, 82, 87, 256, 284-285, 288
|
|
|
|
All step text uses the ``r2cont-`` prefix to avoid collisions with
|
|
existing step definitions in container_and_repository_coverage_steps.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from behave import given, then, when
|
|
from behave.runner import Context
|
|
|
|
# -------------------------------------------------------------------
|
|
# Helpers
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
def _save_env(context: Context, key: str) -> None:
|
|
"""Save an environment variable for later restoration."""
|
|
if not hasattr(context, "_r2cont_saved_env"):
|
|
context._r2cont_saved_env = {}
|
|
if key not in context._r2cont_saved_env:
|
|
context._r2cont_saved_env[key] = os.environ.get(key)
|
|
|
|
|
|
def _restore_env(context: Context) -> None:
|
|
"""Restore all saved environment variables."""
|
|
saved: dict[str, str | None] = getattr(context, "_r2cont_saved_env", {})
|
|
for key, original in saved.items():
|
|
if original is None:
|
|
os.environ.pop(key, None)
|
|
else:
|
|
os.environ[key] = original
|
|
|
|
|
|
def _register_cleanup(context: Context, fn: Any) -> None:
|
|
"""Register a cleanup callable on the behave context."""
|
|
if not hasattr(context, "_r2cont_cleanups"):
|
|
context._r2cont_cleanups = []
|
|
context._r2cont_cleanups.append(fn)
|
|
|
|
|
|
def _run_cleanups(context: Context) -> None:
|
|
"""Run all registered cleanups (called in after_scenario hook or manually)."""
|
|
for fn in getattr(context, "_r2cont_cleanups", []):
|
|
fn()
|
|
context._r2cont_cleanups = []
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Given: get_ai_provider setup
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("r2cont- the mock AI environment flag is disabled")
|
|
def step_disable_mock_ai(context: Context) -> None:
|
|
"""Ensure CLEVERAGENTS_TESTING_USE_MOCK_AI is not set."""
|
|
_save_env(context, "CLEVERAGENTS_TESTING_USE_MOCK_AI")
|
|
os.environ.pop("CLEVERAGENTS_TESTING_USE_MOCK_AI", None)
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
@given("r2cont- a provider registry with no configured providers")
|
|
def step_registry_no_providers(context: Context) -> None:
|
|
"""Create a mock ProviderRegistry that returns no configured providers."""
|
|
mock_registry = MagicMock()
|
|
mock_registry.get_configured_providers.return_value = []
|
|
context.r2cont_registry = mock_registry
|
|
|
|
|
|
@given("r2cont- a provider registry with configured providers")
|
|
def step_registry_with_providers(context: Context) -> None:
|
|
"""Create a mock ProviderRegistry that reports configured providers."""
|
|
mock_registry = MagicMock()
|
|
mock_registry.get_configured_providers.return_value = [MagicMock()]
|
|
mock_provider = MagicMock()
|
|
mock_registry.create_ai_provider.return_value = mock_provider
|
|
context.r2cont_registry = mock_registry
|
|
context.r2cont_expected_provider = mock_provider
|
|
|
|
|
|
@given("r2cont- the mock AI environment flag is enabled")
|
|
def step_enable_mock_ai(context: Context) -> None:
|
|
"""Set the env flag to trigger mock AI loading."""
|
|
_save_env(context, "CLEVERAGENTS_TESTING_USE_MOCK_AI")
|
|
os.environ["CLEVERAGENTS_TESTING_USE_MOCK_AI"] = "true"
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
@given("r2cont- the features path is already on sys.path")
|
|
def step_features_path_on_syspath(context: Context) -> None:
|
|
"""Ensure the features directory IS on sys.path before calling get_ai_provider."""
|
|
from pathlib import Path as _Path
|
|
|
|
# Compute the features path the same way container.py does:
|
|
# Path(__file__).parent.parent.parent.parent / "features"
|
|
# where __file__ is src/cleveragents/application/container.py
|
|
from cleveragents.application.container import __file__ as container_file
|
|
|
|
feat_path = str(_Path(container_file).parent.parent.parent.parent / "features")
|
|
context.r2cont_features_path = feat_path
|
|
|
|
if feat_path not in sys.path:
|
|
sys.path.insert(0, feat_path)
|
|
|
|
# Record the count so we can verify no duplicates were added
|
|
context.r2cont_features_path_count_before = sys.path.count(feat_path)
|
|
|
|
def cleanup() -> None:
|
|
# Remove any extra entries we may have added, restoring original count
|
|
while sys.path.count(feat_path) > context.r2cont_features_path_count_before:
|
|
sys.path.remove(feat_path)
|
|
|
|
_register_cleanup(context, cleanup)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Given: get_database_url env var setup
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("r2cont- no database URL environment variables are set")
|
|
def step_clear_all_db_env_vars(context: Context) -> None:
|
|
"""Remove both database URL env vars."""
|
|
for key in ("CLEVERAGENTS_DATABASE_URL", "CLEVERAGENTS_TEST_DATABASE_URL"):
|
|
_save_env(context, key)
|
|
os.environ.pop(key, None)
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
@given('r2cont- the env var "{key}" is set to empty')
|
|
def step_set_env_var_empty(context: Context, key: str) -> None:
|
|
"""Set a specific environment variable to the empty string."""
|
|
_save_env(context, key)
|
|
os.environ[key] = ""
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
@given('r2cont- the env var "{key}" is set to "{value}"')
|
|
def step_set_env_var(context: Context, key: str, value: str) -> None:
|
|
"""Set a specific environment variable."""
|
|
_save_env(context, key)
|
|
os.environ[key] = value
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
@given('r2cont- the env var "{key}" is cleared')
|
|
def step_clear_env_var(context: Context, key: str) -> None:
|
|
"""Clear a specific environment variable."""
|
|
_save_env(context, key)
|
|
os.environ.pop(key, None)
|
|
_register_cleanup(context, lambda: _restore_env(context))
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Given: Container singleton setup
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@given("r2cont- the global container has been reset")
|
|
def step_reset_global_container(context: Context) -> None:
|
|
"""Reset the global container singleton to None."""
|
|
from cleveragents.application.container import reset_container
|
|
|
|
reset_container()
|
|
_register_cleanup(context, reset_container)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: get_ai_provider
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I call get_ai_provider with the mocked registry")
|
|
def step_call_get_ai_provider_mocked(context: Context) -> None:
|
|
"""Call get_ai_provider passing the mock settings and registry."""
|
|
from cleveragents.application.container import get_ai_provider
|
|
|
|
mock_settings = MagicMock()
|
|
# Ensure Settings.mock_providers is explicitly False so the non-mock
|
|
# branch is exercised (MagicMock auto-attributes are truthy).
|
|
mock_settings.mock_providers = False
|
|
context.r2cont_ai_result = get_ai_provider(
|
|
settings=mock_settings,
|
|
provider_registry=context.r2cont_registry,
|
|
)
|
|
|
|
|
|
@when("r2cont- I call get_ai_provider without registry args")
|
|
def step_call_get_ai_provider_default(context: Context) -> None:
|
|
"""Call get_ai_provider using defaults (will trigger mock AI path)."""
|
|
from cleveragents.application.container import get_ai_provider
|
|
|
|
context.r2cont_ai_result = get_ai_provider()
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: _build_project_resource_link_repo
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I build a project resource link repo with an in-memory database")
|
|
def step_build_project_resource_link_repo(context: Context) -> None:
|
|
"""Call _build_project_resource_link_repo with sqlite memory URL."""
|
|
from cleveragents.application.container import _build_project_resource_link_repo
|
|
|
|
context.r2cont_link_repo = _build_project_resource_link_repo("sqlite:///:memory:")
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: get_database_url
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I call get_database_url")
|
|
def step_call_get_database_url(context: Context) -> None:
|
|
"""Call get_database_url and store the result."""
|
|
from cleveragents.application.container import get_database_url
|
|
|
|
context.r2cont_db_url = get_database_url()
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: get_container
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I call get_container twice")
|
|
def step_call_get_container_twice(context: Context) -> None:
|
|
"""Call get_container twice and store both references."""
|
|
from cleveragents.application.container import get_container
|
|
|
|
context.r2cont_container_first = get_container()
|
|
context.r2cont_container_second = get_container()
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: override_providers - non-existent name
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I override a provider with a non-existent name")
|
|
def step_override_nonexistent(context: Context) -> None:
|
|
"""Call override_providers with a name that doesn't exist on Container."""
|
|
from cleveragents.application.container import override_providers
|
|
|
|
context.r2cont_override_error = None
|
|
try:
|
|
override_providers(this_provider_does_not_exist="some_value")
|
|
except Exception as exc:
|
|
context.r2cont_override_error = exc
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# When: override_providers - non-database_url provider
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@when("r2cont- I override the settings provider with a mock value")
|
|
def step_override_settings(context: Context) -> None:
|
|
"""Override the 'settings' provider (not database_url) to hit the else branch."""
|
|
from cleveragents.application.container import override_providers
|
|
|
|
mock_settings = MagicMock()
|
|
mock_settings._r2cont_marker = "mocked_settings"
|
|
context.r2cont_mock_settings = mock_settings
|
|
override_providers(settings=mock_settings)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Then: get_ai_provider assertions
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then("r2cont- the ai provider result should be None")
|
|
def step_ai_provider_none(context: Context) -> None:
|
|
"""Assert get_ai_provider returned None."""
|
|
assert context.r2cont_ai_result is None, (
|
|
f"Expected None, got {context.r2cont_ai_result}"
|
|
)
|
|
|
|
|
|
@then("r2cont- the ai provider result should not be None")
|
|
def step_ai_provider_not_none(context: Context) -> None:
|
|
"""Assert get_ai_provider returned a provider instance."""
|
|
assert context.r2cont_ai_result is not None, "Expected a provider, got None"
|
|
assert context.r2cont_ai_result is context.r2cont_expected_provider
|
|
|
|
|
|
@then("r2cont- the features path should appear exactly once on sys.path")
|
|
def step_features_path_no_duplicate(context: Context) -> None:
|
|
"""Verify the features path was not duplicated on sys.path."""
|
|
count = sys.path.count(context.r2cont_features_path)
|
|
assert count == context.r2cont_features_path_count_before, (
|
|
f"Expected features path count {context.r2cont_features_path_count_before}, "
|
|
f"got {count}"
|
|
)
|
|
_run_cleanups(context)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Then: _build_project_resource_link_repo assertions
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then("r2cont- the result should be a ProjectResourceLinkRepository instance")
|
|
def step_verify_link_repo_type(context: Context) -> None:
|
|
"""Assert the returned object is a ProjectResourceLinkRepository."""
|
|
from cleveragents.infrastructure.database.repositories import (
|
|
ProjectResourceLinkRepository,
|
|
)
|
|
|
|
assert isinstance(context.r2cont_link_repo, ProjectResourceLinkRepository), (
|
|
f"Expected ProjectResourceLinkRepository, "
|
|
f"got {type(context.r2cont_link_repo).__name__}"
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Then: get_database_url assertions
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then('r2cont- the database URL should start with "{prefix}"')
|
|
def step_db_url_starts_with(context: Context, prefix: str) -> None:
|
|
"""Assert the database URL starts with the given prefix."""
|
|
assert context.r2cont_db_url.startswith(prefix), (
|
|
f"Expected URL starting with '{prefix}', got '{context.r2cont_db_url}'"
|
|
)
|
|
_run_cleanups(context)
|
|
|
|
|
|
@then('r2cont- the database URL should be "{expected}"')
|
|
def step_db_url_exact(context: Context, expected: str) -> None:
|
|
"""Assert the database URL matches exactly."""
|
|
assert context.r2cont_db_url == expected, (
|
|
f"Expected '{expected}', got '{context.r2cont_db_url}'"
|
|
)
|
|
_run_cleanups(context)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Then: get_container assertions
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then("r2cont- both calls should return the same container instance")
|
|
def step_same_container(context: Context) -> None:
|
|
"""Assert both get_container calls returned the same object."""
|
|
assert context.r2cont_container_first is context.r2cont_container_second, (
|
|
"Expected the same container instance on both calls"
|
|
)
|
|
_run_cleanups(context)
|
|
|
|
|
|
# -------------------------------------------------------------------
|
|
# Then: override_providers assertions
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
@then("r2cont- no error should have been raised")
|
|
def step_no_override_error(context: Context) -> None:
|
|
"""Assert no exception was raised during override_providers."""
|
|
assert context.r2cont_override_error is None, (
|
|
f"Unexpected error: {context.r2cont_override_error}"
|
|
)
|
|
_run_cleanups(context)
|
|
|
|
|
|
@then("r2cont- the settings provider should return the mock value")
|
|
def step_settings_overridden(context: Context) -> None:
|
|
"""Assert the settings provider now returns our mock."""
|
|
from cleveragents.application.container import get_container
|
|
|
|
container = get_container()
|
|
result = container.settings()
|
|
assert result is context.r2cont_mock_settings, (
|
|
f"Expected mocked settings, got {result}"
|
|
)
|
|
_run_cleanups(context)
|