Files
cleveragents-core/features/steps/container_coverage_r2_steps.py
freemo a074b4846f
CI / benchmark-publish (pull_request) Has been skipped
CI / lint (pull_request) Successful in 15s
CI / build (pull_request) Successful in 19s
CI / quality (pull_request) Successful in 28s
CI / security (pull_request) Successful in 50s
CI / typecheck (pull_request) Successful in 56s
CI / integration_tests (pull_request) Successful in 4m51s
CI / unit_tests (pull_request) Successful in 19m29s
CI / docker (pull_request) Successful in 39s
CI / benchmark-regression (pull_request) Successful in 26m10s
CI / coverage (pull_request) Successful in 47m42s
CI / lint (push) Successful in 13s
CI / quality (push) Successful in 17s
CI / build (push) Successful in 23s
CI / typecheck (push) Successful in 30s
CI / security (push) Successful in 30s
CI / benchmark-regression (push) Has been skipped
CI / integration_tests (push) Successful in 2m52s
CI / unit_tests (push) Successful in 10m8s
CI / docker (push) Successful in 1m19s
CI / benchmark-publish (push) Successful in 11m54s
CI / coverage (push) Failing after 39m51s
fix(provider): remove FakeListLLM defaults
Remove FakeListLLM as a silent fallback in agent graph constructors
(plan_generation.py, context_analysis.py, auto_debug.py). All three now
raise ValueError when llm=None, making missing-provider errors explicit.

Add Settings.mock_providers flag and validate_provider_availability()
method. Update container.get_ai_provider() to check Settings.mock_providers
first, with env-var fallback for backward compatibility.

Add resolve_provider_by_name() helper to the provider registry and export
it from cleveragents.providers. Add structlog trace logging to
ProviderRegistry.get_default_provider_type() to record selection reasoning.

Update all existing behave step files, robot tests, and benchmarks that
relied on the implicit FakeListLLM default to pass an explicit LLM
instance instead.

Add new BDD tests (features/provider_fixes.feature with 17 scenarios),
Robot Framework integration tests (robot/provider_detection_smoke.robot),
and ASV benchmarks (benchmarks/provider_selection_bench.py).

ISSUES CLOSED: #323
2026-02-27 09:47:10 -05:00

392 lines
15 KiB
Python

"""Step definitions for container_coverage_r2.feature.
Covers remaining uncovered lines and partial branches in
``cleveragents.application.container``:
- Lines 66-67, 69: get_ai_provider non-mock paths
- Lines 125-130: _build_project_resource_link_repo
- Partial branches at lines 51, 57, 82, 87, 256, 284-285, 288
All step text uses the ``r2cont-`` prefix to avoid collisions with
existing step definitions in container_and_repository_coverage_steps.py.
"""
from __future__ import annotations
import os
import sys
from typing import Any
from unittest.mock import MagicMock
from behave import given, then, when
from behave.runner import Context
# -------------------------------------------------------------------
# Helpers
# -------------------------------------------------------------------
def _save_env(context: Context, key: str) -> None:
"""Save an environment variable for later restoration."""
if not hasattr(context, "_r2cont_saved_env"):
context._r2cont_saved_env = {}
if key not in context._r2cont_saved_env:
context._r2cont_saved_env[key] = os.environ.get(key)
def _restore_env(context: Context) -> None:
"""Restore all saved environment variables."""
saved: dict[str, str | None] = getattr(context, "_r2cont_saved_env", {})
for key, original in saved.items():
if original is None:
os.environ.pop(key, None)
else:
os.environ[key] = original
def _register_cleanup(context: Context, fn: Any) -> None:
"""Register a cleanup callable on the behave context."""
if not hasattr(context, "_r2cont_cleanups"):
context._r2cont_cleanups = []
context._r2cont_cleanups.append(fn)
def _run_cleanups(context: Context) -> None:
"""Run all registered cleanups (called in after_scenario hook or manually)."""
for fn in getattr(context, "_r2cont_cleanups", []):
fn()
context._r2cont_cleanups = []
# -------------------------------------------------------------------
# Given: get_ai_provider setup
# -------------------------------------------------------------------
@given("r2cont- the mock AI environment flag is disabled")
def step_disable_mock_ai(context: Context) -> None:
"""Ensure CLEVERAGENTS_TESTING_USE_MOCK_AI is not set."""
_save_env(context, "CLEVERAGENTS_TESTING_USE_MOCK_AI")
os.environ.pop("CLEVERAGENTS_TESTING_USE_MOCK_AI", None)
_register_cleanup(context, lambda: _restore_env(context))
@given("r2cont- a provider registry with no configured providers")
def step_registry_no_providers(context: Context) -> None:
"""Create a mock ProviderRegistry that returns no configured providers."""
mock_registry = MagicMock()
mock_registry.get_configured_providers.return_value = []
context.r2cont_registry = mock_registry
@given("r2cont- a provider registry with configured providers")
def step_registry_with_providers(context: Context) -> None:
"""Create a mock ProviderRegistry that reports configured providers."""
mock_registry = MagicMock()
mock_registry.get_configured_providers.return_value = [MagicMock()]
mock_provider = MagicMock()
mock_registry.create_ai_provider.return_value = mock_provider
context.r2cont_registry = mock_registry
context.r2cont_expected_provider = mock_provider
@given("r2cont- the mock AI environment flag is enabled")
def step_enable_mock_ai(context: Context) -> None:
"""Set the env flag to trigger mock AI loading."""
_save_env(context, "CLEVERAGENTS_TESTING_USE_MOCK_AI")
os.environ["CLEVERAGENTS_TESTING_USE_MOCK_AI"] = "true"
_register_cleanup(context, lambda: _restore_env(context))
@given("r2cont- the features path is already on sys.path")
def step_features_path_on_syspath(context: Context) -> None:
"""Ensure the features directory IS on sys.path before calling get_ai_provider."""
from pathlib import Path as _Path
# Compute the features path the same way container.py does:
# Path(__file__).parent.parent.parent.parent / "features"
# where __file__ is src/cleveragents/application/container.py
from cleveragents.application.container import __file__ as container_file
feat_path = str(_Path(container_file).parent.parent.parent.parent / "features")
context.r2cont_features_path = feat_path
if feat_path not in sys.path:
sys.path.insert(0, feat_path)
# Record the count so we can verify no duplicates were added
context.r2cont_features_path_count_before = sys.path.count(feat_path)
def cleanup() -> None:
# Remove any extra entries we may have added, restoring original count
while sys.path.count(feat_path) > context.r2cont_features_path_count_before:
sys.path.remove(feat_path)
_register_cleanup(context, cleanup)
# -------------------------------------------------------------------
# Given: get_database_url env var setup
# -------------------------------------------------------------------
@given("r2cont- no database URL environment variables are set")
def step_clear_all_db_env_vars(context: Context) -> None:
"""Remove both database URL env vars."""
for key in ("CLEVERAGENTS_DATABASE_URL", "CLEVERAGENTS_TEST_DATABASE_URL"):
_save_env(context, key)
os.environ.pop(key, None)
_register_cleanup(context, lambda: _restore_env(context))
@given('r2cont- the env var "{key}" is set to empty')
def step_set_env_var_empty(context: Context, key: str) -> None:
"""Set a specific environment variable to the empty string."""
_save_env(context, key)
os.environ[key] = ""
_register_cleanup(context, lambda: _restore_env(context))
@given('r2cont- the env var "{key}" is set to "{value}"')
def step_set_env_var(context: Context, key: str, value: str) -> None:
"""Set a specific environment variable."""
_save_env(context, key)
os.environ[key] = value
_register_cleanup(context, lambda: _restore_env(context))
@given('r2cont- the env var "{key}" is cleared')
def step_clear_env_var(context: Context, key: str) -> None:
"""Clear a specific environment variable."""
_save_env(context, key)
os.environ.pop(key, None)
_register_cleanup(context, lambda: _restore_env(context))
# -------------------------------------------------------------------
# Given: Container singleton setup
# -------------------------------------------------------------------
@given("r2cont- the global container has been reset")
def step_reset_global_container(context: Context) -> None:
"""Reset the global container singleton to None."""
from cleveragents.application.container import reset_container
reset_container()
_register_cleanup(context, reset_container)
# -------------------------------------------------------------------
# When: get_ai_provider
# -------------------------------------------------------------------
@when("r2cont- I call get_ai_provider with the mocked registry")
def step_call_get_ai_provider_mocked(context: Context) -> None:
"""Call get_ai_provider passing the mock settings and registry."""
from cleveragents.application.container import get_ai_provider
mock_settings = MagicMock()
# Ensure Settings.mock_providers is explicitly False so the non-mock
# branch is exercised (MagicMock auto-attributes are truthy).
mock_settings.mock_providers = False
context.r2cont_ai_result = get_ai_provider(
settings=mock_settings,
provider_registry=context.r2cont_registry,
)
@when("r2cont- I call get_ai_provider without registry args")
def step_call_get_ai_provider_default(context: Context) -> None:
"""Call get_ai_provider using defaults (will trigger mock AI path)."""
from cleveragents.application.container import get_ai_provider
context.r2cont_ai_result = get_ai_provider()
# -------------------------------------------------------------------
# When: _build_project_resource_link_repo
# -------------------------------------------------------------------
@when("r2cont- I build a project resource link repo with an in-memory database")
def step_build_project_resource_link_repo(context: Context) -> None:
"""Call _build_project_resource_link_repo with sqlite memory URL."""
from cleveragents.application.container import _build_project_resource_link_repo
context.r2cont_link_repo = _build_project_resource_link_repo("sqlite:///:memory:")
# -------------------------------------------------------------------
# When: get_database_url
# -------------------------------------------------------------------
@when("r2cont- I call get_database_url")
def step_call_get_database_url(context: Context) -> None:
"""Call get_database_url and store the result."""
from cleveragents.application.container import get_database_url
context.r2cont_db_url = get_database_url()
# -------------------------------------------------------------------
# When: get_container
# -------------------------------------------------------------------
@when("r2cont- I call get_container twice")
def step_call_get_container_twice(context: Context) -> None:
"""Call get_container twice and store both references."""
from cleveragents.application.container import get_container
context.r2cont_container_first = get_container()
context.r2cont_container_second = get_container()
# -------------------------------------------------------------------
# When: override_providers - non-existent name
# -------------------------------------------------------------------
@when("r2cont- I override a provider with a non-existent name")
def step_override_nonexistent(context: Context) -> None:
"""Call override_providers with a name that doesn't exist on Container."""
from cleveragents.application.container import override_providers
context.r2cont_override_error = None
try:
override_providers(this_provider_does_not_exist="some_value")
except Exception as exc:
context.r2cont_override_error = exc
# -------------------------------------------------------------------
# When: override_providers - non-database_url provider
# -------------------------------------------------------------------
@when("r2cont- I override the settings provider with a mock value")
def step_override_settings(context: Context) -> None:
"""Override the 'settings' provider (not database_url) to hit the else branch."""
from cleveragents.application.container import override_providers
mock_settings = MagicMock()
mock_settings._r2cont_marker = "mocked_settings"
context.r2cont_mock_settings = mock_settings
override_providers(settings=mock_settings)
# -------------------------------------------------------------------
# Then: get_ai_provider assertions
# -------------------------------------------------------------------
@then("r2cont- the ai provider result should be None")
def step_ai_provider_none(context: Context) -> None:
"""Assert get_ai_provider returned None."""
assert context.r2cont_ai_result is None, (
f"Expected None, got {context.r2cont_ai_result}"
)
@then("r2cont- the ai provider result should not be None")
def step_ai_provider_not_none(context: Context) -> None:
"""Assert get_ai_provider returned a provider instance."""
assert context.r2cont_ai_result is not None, "Expected a provider, got None"
assert context.r2cont_ai_result is context.r2cont_expected_provider
@then("r2cont- the features path should appear exactly once on sys.path")
def step_features_path_no_duplicate(context: Context) -> None:
"""Verify the features path was not duplicated on sys.path."""
count = sys.path.count(context.r2cont_features_path)
assert count == context.r2cont_features_path_count_before, (
f"Expected features path count {context.r2cont_features_path_count_before}, "
f"got {count}"
)
_run_cleanups(context)
# -------------------------------------------------------------------
# Then: _build_project_resource_link_repo assertions
# -------------------------------------------------------------------
@then("r2cont- the result should be a ProjectResourceLinkRepository instance")
def step_verify_link_repo_type(context: Context) -> None:
"""Assert the returned object is a ProjectResourceLinkRepository."""
from cleveragents.infrastructure.database.repositories import (
ProjectResourceLinkRepository,
)
assert isinstance(context.r2cont_link_repo, ProjectResourceLinkRepository), (
f"Expected ProjectResourceLinkRepository, "
f"got {type(context.r2cont_link_repo).__name__}"
)
# -------------------------------------------------------------------
# Then: get_database_url assertions
# -------------------------------------------------------------------
@then('r2cont- the database URL should start with "{prefix}"')
def step_db_url_starts_with(context: Context, prefix: str) -> None:
"""Assert the database URL starts with the given prefix."""
assert context.r2cont_db_url.startswith(prefix), (
f"Expected URL starting with '{prefix}', got '{context.r2cont_db_url}'"
)
_run_cleanups(context)
@then('r2cont- the database URL should be "{expected}"')
def step_db_url_exact(context: Context, expected: str) -> None:
"""Assert the database URL matches exactly."""
assert context.r2cont_db_url == expected, (
f"Expected '{expected}', got '{context.r2cont_db_url}'"
)
_run_cleanups(context)
# -------------------------------------------------------------------
# Then: get_container assertions
# -------------------------------------------------------------------
@then("r2cont- both calls should return the same container instance")
def step_same_container(context: Context) -> None:
"""Assert both get_container calls returned the same object."""
assert context.r2cont_container_first is context.r2cont_container_second, (
"Expected the same container instance on both calls"
)
_run_cleanups(context)
# -------------------------------------------------------------------
# Then: override_providers assertions
# -------------------------------------------------------------------
@then("r2cont- no error should have been raised")
def step_no_override_error(context: Context) -> None:
"""Assert no exception was raised during override_providers."""
assert context.r2cont_override_error is None, (
f"Unexpected error: {context.r2cont_override_error}"
)
_run_cleanups(context)
@then("r2cont- the settings provider should return the mock value")
def step_settings_overridden(context: Context) -> None:
"""Assert the settings provider now returns our mock."""
from cleveragents.application.container import get_container
container = get_container()
result = container.settings()
assert result is context.r2cont_mock_settings, (
f"Expected mocked settings, got {result}"
)
_run_cleanups(context)