Test: Added another 1% test coverage

This commit is contained in:
2025-12-06 18:36:29 -05:00
parent 480d77a18e
commit 2eb4e08c54
4 changed files with 332 additions and 0 deletions
@@ -33,3 +33,37 @@ Feature: LangChain chat provider coverage
When the provider generates changes and retries exhaust with nested errors
Then the provider response should capture the nested retry failure
And the progress callback should end at 100 percent even on failure
@coverage @langchain @no_progress
Scenario: LangChain provider handles invalid state data without a progress callback
Given a LangChain chat provider is configured with a fake LangChain graph
When the provider generates changes without a progress callback and the graph returns invalid response data
Then the provider response should surface the state error without changes
And the progress callback should not receive updates
@coverage @langchain @streaming @errors
Scenario: LangChain provider records partial streaming progress when the stream fails
Given a LangChain chat provider is configured with a fake LangChain graph
When the provider streams changes and the stream raises an exception after unknown node events
Then the provider response should capture the graph failure
And the progress callback should capture partial streaming progress before failure
@coverage @langchain @retry
Scenario: LangChain provider reports failure when retry runner yields no attempts
Given a LangChain chat provider is configured with a fake LangChain graph
When the provider generates changes and the retry runner yields no attempts
Then the provider response should capture the graph failure
And the progress callback should end at 100 percent even on failure
@coverage @langchain @retry
Scenario Outline: LangChain provider falls back to retry error strings when metadata is incomplete
Given a LangChain chat provider is configured with a fake LangChain graph
When the provider generates changes and the retry metadata is "<variant>"
Then the provider response should capture the nested retry failure
And the progress callback should end at 100 percent even on failure
Examples:
| variant |
| missing-last-attempt |
| non-callable-exception |
| non-exception-result |
+31
View File
@@ -58,3 +58,34 @@ Feature: Settings runtime helpers
Scenario: LangSmith config builder returns None when disabled
When I build the LangSmith config
Then the LangSmith config should be absent
Scenario: Relative vector store path resolves against cwd
Given the vector store path override is "relative/vector"
When I resolve the vector store path
Then the resolved vector store path should equal the override under the current working directory
Scenario: LangSmith validation surfaces missing credentials
Given LangSmith is enabled without credentials
When I validate the LangSmith configuration
Then LangSmith validation should fail with 2 errors
Scenario: LangSmith endpoint synchronization updates LangChain environment
Given LangSmith is configured with API key "abc123", project "coverage-runner", and endpoint "https://trace.local"
When I check if LangSmith is enabled
Then the "LANGCHAIN_ENDPOINT" environment variable should equal "https://trace.local"
Scenario: Provider lookup trims api key suffix before resolving
Given the environment variable "OPENAI_API_KEY" is set to "env-key"
When I check if "openai_api_key" provider is configured
Then has_provider_configured should be True for "openai_api_key"
Scenario: Provider lookup falls back to direct map entries
Given the environment variable "HF_TOKEN" is set to "hf-secret"
When I check if "hf_token" provider is configured
Then has_provider_configured should be True for "hf_token"
Scenario: Custom provider candidate resolves env mapping
Given a temporary provider "custom" uses env "CUSTOM_API_KEY"
And the environment variable "CUSTOM_API_KEY" is set to "dynamic-key"
When I check if "custom_api_key" provider is configured using a constructed instance
Then has_provider_configured should be True for "custom_api_key"
@@ -99,6 +99,34 @@ def step_provider_generates_exception(context):
context.failure_message = failure_message
@when(
"the provider generates changes without a progress callback and the graph returns invalid response data"
)
def step_provider_invalid_state_without_progress(context):
context.llm_instance.get_num_tokens = None
context.contexts = [MagicMock(spec=Context)]
context.contexts[0].content = ""
invalid_state = {
"generated_changes": "not-a-list",
"validation_result": "not-a-dict",
"error": "Graph returned invalid payload",
}
context.state_error_message = invalid_state["error"]
with patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
) as graph_cls:
mock_graph = graph_cls.return_value
mock_graph.invoke.return_value = invalid_state
context.graph_instance = mock_graph
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
progress_callback=None,
)
@when("the provider streams changes with incremental graph events")
def step_provider_streams_changes(context):
context.provider = context.create_provider(supports_streaming=True)
@@ -143,6 +171,36 @@ def step_provider_streams_changes(context):
context.expected_token_count = 321
@when(
"the provider streams changes and the stream raises an exception after unknown node events"
)
def step_provider_stream_failure_with_unknown_nodes(context):
context.provider = context.create_provider(supports_streaming=True)
failure_message = "Streaming interrupted by upstream failure"
streaming_events = [{"load_context": {}}, {"mystery_node": "unexpected-payload"}]
context.llm_instance.get_num_tokens.return_value = None
def _failing_stream():
for event in streaming_events:
yield event
raise RuntimeError(failure_message)
with patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
) as graph_cls:
mock_graph = graph_cls.return_value
mock_graph.stream.return_value = _failing_stream()
context.graph_instance = mock_graph
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
progress_callback=context.progress_callback,
)
context.failure_message = failure_message
@when("the provider generates changes and retries exhaust with nested errors")
def step_provider_generates_retry_failure(context):
final_message = "All retry attempts failed"
@@ -174,6 +232,79 @@ def step_provider_generates_retry_failure(context):
context.failure_message = final_message
@when('the provider generates changes and the retry metadata is "{variant}"')
def step_provider_incomplete_retry_metadata(context, variant):
class RetryError(Exception):
pass
retry_error = RetryError(f"Incomplete retry metadata: {variant}")
if variant == "missing-last-attempt":
retry_error.last_attempt = None
elif variant == "non-callable-exception":
class _Attempt:
exception = "not-callable"
retry_error.last_attempt = _Attempt()
elif variant == "non-exception-result":
class _Attempt:
def exception(self):
return "not-exception"
retry_error.last_attempt = _Attempt()
else: # pragma: no cover - defensive guard
raise AssertionError(f"Unknown retry metadata variant: {variant}")
with patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
) as graph_cls:
mock_graph = graph_cls.return_value
mock_graph.invoke.side_effect = retry_error
context.graph_instance = mock_graph
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
progress_callback=context.progress_callback,
)
context.failure_message = str(retry_error)
@when("the provider generates changes and the retry runner yields no attempts")
def step_provider_retry_runner_without_attempts(context):
failure_message = "Retries exhausted while invoking plan generation graph"
class _EmptyRetrying:
def __init__(self, *_, **__):
pass
def __iter__(self):
return iter(())
with (
patch(
"cleveragents.providers.llm.langchain_chat_provider.Retrying",
_EmptyRetrying,
),
patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
) as graph_cls,
):
mock_graph = graph_cls.return_value
context.graph_instance = mock_graph
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
progress_callback=context.progress_callback,
)
context.failure_message = failure_message
@then(
"the provider response should contain the generated change data and validation error"
)
@@ -189,6 +320,19 @@ def step_assert_response_contains_validation_error(context):
assert context.requested_models == ["test-model"]
@then("the provider response should surface the state error without changes")
def step_assert_state_error_without_changes(context):
assert context.response is not None
assert context.response.changes == []
assert context.response.error_message == context.state_error_message
assert context.response.token_count == 0
@then("the progress callback should not receive updates")
def step_assert_no_progress_updates(context):
assert context.progress_updates == []
@then("the progress callback should record the LangChain workflow milestones")
def step_assert_progress_milestones(context):
assert context.progress_updates == [5, 90, 100]
@@ -226,6 +370,11 @@ def step_assert_streaming_progress(context):
assert context.progress_updates == [5, 15, 40, 70, 90, 100]
@then("the progress callback should capture partial streaming progress before failure")
def step_assert_streaming_partial_progress(context):
assert context.progress_updates == [5, 15, 100, 100]
@then("the LangChain graph stream should receive the thread-aware configuration")
def step_assert_stream_call(context):
call_args, call_kwargs = context.stream_call_args
+118
View File
@@ -173,6 +173,33 @@ def step_check_storage_path(context, expected):
assert str(context.storage_path) == expected
@given('the vector store path override is "{path_value}"')
def step_set_vector_store_override(context, path_value):
"""Record the vector store path override for later resolution."""
context.vector_store_override = path_value
@when("I resolve the vector store path")
def step_resolve_vector_store_path(context):
"""Resolve the configured vector store path to an absolute location."""
override = getattr(context, "vector_store_override", None)
assert override is not None, "Vector store path override must be provided first."
Settings._instance = None
settings = Settings.model_construct(vector_store_path=Path(override))
context.resolved_vector_store_path = settings.resolve_vector_store_path()
@then(
"the resolved vector store path should equal the override under the current working directory"
)
def step_verify_resolved_vector_store_path(context):
"""Ensure the vector store path is resolved relative to the current working directory."""
override = getattr(context, "vector_store_override", None)
assert override is not None, "Vector store path override must be provided first."
expected = (Path.cwd() / Path(override)).resolve()
assert context.resolved_vector_store_path == expected
@given('the environment is set to "{env}"')
def step_set_environment(context, env):
"""Set the environment."""
@@ -246,6 +273,16 @@ def step_check_provider_configured(context, provider):
)
@when('I check if "{provider}" provider is configured using a constructed instance')
def step_check_provider_configured_constructed(context, provider):
"""Check provider configuration using a lightweight constructed settings instance."""
Settings._instance = None
settings = Settings.model_construct()
context.provider_configured = settings.has_provider_configured(
provider.lower().replace(" ", "")
)
@when('I clear the explicit "{provider}" key and recheck configuration')
def step_clear_explicit_key_and_recheck(context, provider):
"""Clear the in-memory provider key and re-evaluate configuration."""
@@ -287,6 +324,30 @@ def step_clear_api_keys(context):
os.environ.pop(key, None)
@given('a temporary provider "{provider}" uses env "{env_var}"')
def step_define_temporary_provider(context, provider, env_var):
"""Temporarily register a provider mapping for provider lookup tests."""
normalized = provider.lower().replace(" ", "_")
if not normalized.endswith("_api_key"):
normalized = f"{normalized}_api_key"
original_map = Settings._PROVIDER_ENV_MAP
original_override = Settings._apply_external_env_overrides
Settings._PROVIDER_ENV_MAP = dict(Settings._PROVIDER_ENV_MAP)
Settings._PROVIDER_ENV_MAP[normalized] = (env_var,)
Settings._apply_external_env_overrides = lambda self: None
if not hasattr(context, "_cleanup_handlers"):
context._cleanup_handlers = []
def cleanup():
Settings._PROVIDER_ENV_MAP = original_map
Settings._apply_external_env_overrides = original_override
context._cleanup_handlers.append(cleanup)
@when("I check if any provider is configured")
def step_check_any_provider(context):
"""Check if any provider is configured."""
@@ -372,3 +433,60 @@ def step_build_langsmith_config(context):
def step_assert_langsmith_config_absent(context):
"""Ensure LangSmith config is not generated when disabled."""
assert context.langsmith_config is None
@given("LangSmith is enabled without credentials")
def step_enable_langsmith_without_credentials(context):
"""Create a settings object with LangSmith enabled but missing key/project."""
Settings._instance = None
context.settings = Settings.model_construct(
langsmith_enabled=True,
langsmith_api_key=None,
langsmith_project=None,
)
@when("I validate the LangSmith configuration")
def step_validate_langsmith_configuration(context):
"""Run validation against the current LangSmith settings."""
assert hasattr(context, "settings"), "Settings must be initialized first."
context.langsmith_validation = context.settings.validate_langsmith_configuration()
@then("LangSmith validation should fail with {error_count:d} errors")
def step_assert_langsmith_validation_errors(context, error_count):
"""Assert the validation result and number of reported errors."""
enabled, errors = context.langsmith_validation
assert enabled is False
assert len(errors) == error_count
@given(
'LangSmith is configured with API key "{api_key}", project "{project}", and endpoint "{endpoint}"'
)
def step_configure_langsmith_with_endpoint(context, api_key, project, endpoint):
"""Construct LangSmith settings with explicit endpoint details."""
Settings._instance = None
context.settings = Settings.model_construct(
langsmith_enabled=True,
langsmith_api_key=api_key,
langsmith_project=project,
langsmith_endpoint=endpoint,
)
@when("I check if LangSmith is enabled")
def step_check_langsmith_enabled(context):
"""Evaluate whether LangSmith is enabled for the current settings."""
assert hasattr(context, "settings"), "Settings must be initialized first."
context.langsmith_enabled_flag = context.settings.is_langsmith_enabled
@then('the "{env_var}" environment variable should equal "{expected}"')
def step_verify_environment_variable(context, env_var, expected):
"""Verify that an environment variable has the expected value."""
assert os.environ.get(env_var) == expected
if not hasattr(context, "env_vars_to_clean"):
context.env_vars_to_clean = []
if env_var not in context.env_vars_to_clean:
context.env_vars_to_clean.append(env_var)