Test: Added another 1% test coverage
This commit is contained in:
@@ -33,3 +33,37 @@ Feature: LangChain chat provider coverage
|
||||
When the provider generates changes and retries exhaust with nested errors
|
||||
Then the provider response should capture the nested retry failure
|
||||
And the progress callback should end at 100 percent even on failure
|
||||
|
||||
@coverage @langchain @no_progress
|
||||
Scenario: LangChain provider handles invalid state data without a progress callback
|
||||
Given a LangChain chat provider is configured with a fake LangChain graph
|
||||
When the provider generates changes without a progress callback and the graph returns invalid response data
|
||||
Then the provider response should surface the state error without changes
|
||||
And the progress callback should not receive updates
|
||||
|
||||
@coverage @langchain @streaming @errors
|
||||
Scenario: LangChain provider records partial streaming progress when the stream fails
|
||||
Given a LangChain chat provider is configured with a fake LangChain graph
|
||||
When the provider streams changes and the stream raises an exception after unknown node events
|
||||
Then the provider response should capture the graph failure
|
||||
And the progress callback should capture partial streaming progress before failure
|
||||
|
||||
@coverage @langchain @retry
|
||||
Scenario: LangChain provider reports failure when retry runner yields no attempts
|
||||
Given a LangChain chat provider is configured with a fake LangChain graph
|
||||
When the provider generates changes and the retry runner yields no attempts
|
||||
Then the provider response should capture the graph failure
|
||||
And the progress callback should end at 100 percent even on failure
|
||||
|
||||
@coverage @langchain @retry
|
||||
Scenario Outline: LangChain provider falls back to retry error strings when metadata is incomplete
|
||||
Given a LangChain chat provider is configured with a fake LangChain graph
|
||||
When the provider generates changes and the retry metadata is "<variant>"
|
||||
Then the provider response should capture the nested retry failure
|
||||
And the progress callback should end at 100 percent even on failure
|
||||
|
||||
Examples:
|
||||
| variant |
|
||||
| missing-last-attempt |
|
||||
| non-callable-exception |
|
||||
| non-exception-result |
|
||||
|
||||
@@ -58,3 +58,34 @@ Feature: Settings runtime helpers
|
||||
Scenario: LangSmith config builder returns None when disabled
|
||||
When I build the LangSmith config
|
||||
Then the LangSmith config should be absent
|
||||
|
||||
Scenario: Relative vector store path resolves against cwd
|
||||
Given the vector store path override is "relative/vector"
|
||||
When I resolve the vector store path
|
||||
Then the resolved vector store path should equal the override under the current working directory
|
||||
|
||||
Scenario: LangSmith validation surfaces missing credentials
|
||||
Given LangSmith is enabled without credentials
|
||||
When I validate the LangSmith configuration
|
||||
Then LangSmith validation should fail with 2 errors
|
||||
|
||||
Scenario: LangSmith endpoint synchronization updates LangChain environment
|
||||
Given LangSmith is configured with API key "abc123", project "coverage-runner", and endpoint "https://trace.local"
|
||||
When I check if LangSmith is enabled
|
||||
Then the "LANGCHAIN_ENDPOINT" environment variable should equal "https://trace.local"
|
||||
|
||||
Scenario: Provider lookup trims api key suffix before resolving
|
||||
Given the environment variable "OPENAI_API_KEY" is set to "env-key"
|
||||
When I check if "openai_api_key" provider is configured
|
||||
Then has_provider_configured should be True for "openai_api_key"
|
||||
|
||||
Scenario: Provider lookup falls back to direct map entries
|
||||
Given the environment variable "HF_TOKEN" is set to "hf-secret"
|
||||
When I check if "hf_token" provider is configured
|
||||
Then has_provider_configured should be True for "hf_token"
|
||||
|
||||
Scenario: Custom provider candidate resolves env mapping
|
||||
Given a temporary provider "custom" uses env "CUSTOM_API_KEY"
|
||||
And the environment variable "CUSTOM_API_KEY" is set to "dynamic-key"
|
||||
When I check if "custom_api_key" provider is configured using a constructed instance
|
||||
Then has_provider_configured should be True for "custom_api_key"
|
||||
|
||||
@@ -99,6 +99,34 @@ def step_provider_generates_exception(context):
|
||||
context.failure_message = failure_message
|
||||
|
||||
|
||||
@when(
|
||||
"the provider generates changes without a progress callback and the graph returns invalid response data"
|
||||
)
|
||||
def step_provider_invalid_state_without_progress(context):
|
||||
context.llm_instance.get_num_tokens = None
|
||||
context.contexts = [MagicMock(spec=Context)]
|
||||
context.contexts[0].content = ""
|
||||
invalid_state = {
|
||||
"generated_changes": "not-a-list",
|
||||
"validation_result": "not-a-dict",
|
||||
"error": "Graph returned invalid payload",
|
||||
}
|
||||
context.state_error_message = invalid_state["error"]
|
||||
|
||||
with patch(
|
||||
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
|
||||
) as graph_cls:
|
||||
mock_graph = graph_cls.return_value
|
||||
mock_graph.invoke.return_value = invalid_state
|
||||
context.graph_instance = mock_graph
|
||||
context.response = context.provider.generate_changes(
|
||||
context.project,
|
||||
context.plan,
|
||||
context.contexts,
|
||||
progress_callback=None,
|
||||
)
|
||||
|
||||
|
||||
@when("the provider streams changes with incremental graph events")
|
||||
def step_provider_streams_changes(context):
|
||||
context.provider = context.create_provider(supports_streaming=True)
|
||||
@@ -143,6 +171,36 @@ def step_provider_streams_changes(context):
|
||||
context.expected_token_count = 321
|
||||
|
||||
|
||||
@when(
|
||||
"the provider streams changes and the stream raises an exception after unknown node events"
|
||||
)
|
||||
def step_provider_stream_failure_with_unknown_nodes(context):
|
||||
context.provider = context.create_provider(supports_streaming=True)
|
||||
failure_message = "Streaming interrupted by upstream failure"
|
||||
streaming_events = [{"load_context": {}}, {"mystery_node": "unexpected-payload"}]
|
||||
context.llm_instance.get_num_tokens.return_value = None
|
||||
|
||||
def _failing_stream():
|
||||
for event in streaming_events:
|
||||
yield event
|
||||
raise RuntimeError(failure_message)
|
||||
|
||||
with patch(
|
||||
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
|
||||
) as graph_cls:
|
||||
mock_graph = graph_cls.return_value
|
||||
mock_graph.stream.return_value = _failing_stream()
|
||||
context.graph_instance = mock_graph
|
||||
context.response = context.provider.generate_changes(
|
||||
context.project,
|
||||
context.plan,
|
||||
context.contexts,
|
||||
progress_callback=context.progress_callback,
|
||||
)
|
||||
|
||||
context.failure_message = failure_message
|
||||
|
||||
|
||||
@when("the provider generates changes and retries exhaust with nested errors")
|
||||
def step_provider_generates_retry_failure(context):
|
||||
final_message = "All retry attempts failed"
|
||||
@@ -174,6 +232,79 @@ def step_provider_generates_retry_failure(context):
|
||||
context.failure_message = final_message
|
||||
|
||||
|
||||
@when('the provider generates changes and the retry metadata is "{variant}"')
|
||||
def step_provider_incomplete_retry_metadata(context, variant):
|
||||
class RetryError(Exception):
|
||||
pass
|
||||
|
||||
retry_error = RetryError(f"Incomplete retry metadata: {variant}")
|
||||
|
||||
if variant == "missing-last-attempt":
|
||||
retry_error.last_attempt = None
|
||||
elif variant == "non-callable-exception":
|
||||
|
||||
class _Attempt:
|
||||
exception = "not-callable"
|
||||
|
||||
retry_error.last_attempt = _Attempt()
|
||||
elif variant == "non-exception-result":
|
||||
|
||||
class _Attempt:
|
||||
def exception(self):
|
||||
return "not-exception"
|
||||
|
||||
retry_error.last_attempt = _Attempt()
|
||||
else: # pragma: no cover - defensive guard
|
||||
raise AssertionError(f"Unknown retry metadata variant: {variant}")
|
||||
|
||||
with patch(
|
||||
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
|
||||
) as graph_cls:
|
||||
mock_graph = graph_cls.return_value
|
||||
mock_graph.invoke.side_effect = retry_error
|
||||
context.graph_instance = mock_graph
|
||||
context.response = context.provider.generate_changes(
|
||||
context.project,
|
||||
context.plan,
|
||||
context.contexts,
|
||||
progress_callback=context.progress_callback,
|
||||
)
|
||||
|
||||
context.failure_message = str(retry_error)
|
||||
|
||||
|
||||
@when("the provider generates changes and the retry runner yields no attempts")
|
||||
def step_provider_retry_runner_without_attempts(context):
|
||||
failure_message = "Retries exhausted while invoking plan generation graph"
|
||||
|
||||
class _EmptyRetrying:
|
||||
def __init__(self, *_, **__):
|
||||
pass
|
||||
|
||||
def __iter__(self):
|
||||
return iter(())
|
||||
|
||||
with (
|
||||
patch(
|
||||
"cleveragents.providers.llm.langchain_chat_provider.Retrying",
|
||||
_EmptyRetrying,
|
||||
),
|
||||
patch(
|
||||
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
|
||||
) as graph_cls,
|
||||
):
|
||||
mock_graph = graph_cls.return_value
|
||||
context.graph_instance = mock_graph
|
||||
context.response = context.provider.generate_changes(
|
||||
context.project,
|
||||
context.plan,
|
||||
context.contexts,
|
||||
progress_callback=context.progress_callback,
|
||||
)
|
||||
|
||||
context.failure_message = failure_message
|
||||
|
||||
|
||||
@then(
|
||||
"the provider response should contain the generated change data and validation error"
|
||||
)
|
||||
@@ -189,6 +320,19 @@ def step_assert_response_contains_validation_error(context):
|
||||
assert context.requested_models == ["test-model"]
|
||||
|
||||
|
||||
@then("the provider response should surface the state error without changes")
|
||||
def step_assert_state_error_without_changes(context):
|
||||
assert context.response is not None
|
||||
assert context.response.changes == []
|
||||
assert context.response.error_message == context.state_error_message
|
||||
assert context.response.token_count == 0
|
||||
|
||||
|
||||
@then("the progress callback should not receive updates")
|
||||
def step_assert_no_progress_updates(context):
|
||||
assert context.progress_updates == []
|
||||
|
||||
|
||||
@then("the progress callback should record the LangChain workflow milestones")
|
||||
def step_assert_progress_milestones(context):
|
||||
assert context.progress_updates == [5, 90, 100]
|
||||
@@ -226,6 +370,11 @@ def step_assert_streaming_progress(context):
|
||||
assert context.progress_updates == [5, 15, 40, 70, 90, 100]
|
||||
|
||||
|
||||
@then("the progress callback should capture partial streaming progress before failure")
|
||||
def step_assert_streaming_partial_progress(context):
|
||||
assert context.progress_updates == [5, 15, 100, 100]
|
||||
|
||||
|
||||
@then("the LangChain graph stream should receive the thread-aware configuration")
|
||||
def step_assert_stream_call(context):
|
||||
call_args, call_kwargs = context.stream_call_args
|
||||
|
||||
@@ -173,6 +173,33 @@ def step_check_storage_path(context, expected):
|
||||
assert str(context.storage_path) == expected
|
||||
|
||||
|
||||
@given('the vector store path override is "{path_value}"')
|
||||
def step_set_vector_store_override(context, path_value):
|
||||
"""Record the vector store path override for later resolution."""
|
||||
context.vector_store_override = path_value
|
||||
|
||||
|
||||
@when("I resolve the vector store path")
|
||||
def step_resolve_vector_store_path(context):
|
||||
"""Resolve the configured vector store path to an absolute location."""
|
||||
override = getattr(context, "vector_store_override", None)
|
||||
assert override is not None, "Vector store path override must be provided first."
|
||||
Settings._instance = None
|
||||
settings = Settings.model_construct(vector_store_path=Path(override))
|
||||
context.resolved_vector_store_path = settings.resolve_vector_store_path()
|
||||
|
||||
|
||||
@then(
|
||||
"the resolved vector store path should equal the override under the current working directory"
|
||||
)
|
||||
def step_verify_resolved_vector_store_path(context):
|
||||
"""Ensure the vector store path is resolved relative to the current working directory."""
|
||||
override = getattr(context, "vector_store_override", None)
|
||||
assert override is not None, "Vector store path override must be provided first."
|
||||
expected = (Path.cwd() / Path(override)).resolve()
|
||||
assert context.resolved_vector_store_path == expected
|
||||
|
||||
|
||||
@given('the environment is set to "{env}"')
|
||||
def step_set_environment(context, env):
|
||||
"""Set the environment."""
|
||||
@@ -246,6 +273,16 @@ def step_check_provider_configured(context, provider):
|
||||
)
|
||||
|
||||
|
||||
@when('I check if "{provider}" provider is configured using a constructed instance')
|
||||
def step_check_provider_configured_constructed(context, provider):
|
||||
"""Check provider configuration using a lightweight constructed settings instance."""
|
||||
Settings._instance = None
|
||||
settings = Settings.model_construct()
|
||||
context.provider_configured = settings.has_provider_configured(
|
||||
provider.lower().replace(" ", "")
|
||||
)
|
||||
|
||||
|
||||
@when('I clear the explicit "{provider}" key and recheck configuration')
|
||||
def step_clear_explicit_key_and_recheck(context, provider):
|
||||
"""Clear the in-memory provider key and re-evaluate configuration."""
|
||||
@@ -287,6 +324,30 @@ def step_clear_api_keys(context):
|
||||
os.environ.pop(key, None)
|
||||
|
||||
|
||||
@given('a temporary provider "{provider}" uses env "{env_var}"')
|
||||
def step_define_temporary_provider(context, provider, env_var):
|
||||
"""Temporarily register a provider mapping for provider lookup tests."""
|
||||
normalized = provider.lower().replace(" ", "_")
|
||||
if not normalized.endswith("_api_key"):
|
||||
normalized = f"{normalized}_api_key"
|
||||
|
||||
original_map = Settings._PROVIDER_ENV_MAP
|
||||
original_override = Settings._apply_external_env_overrides
|
||||
|
||||
Settings._PROVIDER_ENV_MAP = dict(Settings._PROVIDER_ENV_MAP)
|
||||
Settings._PROVIDER_ENV_MAP[normalized] = (env_var,)
|
||||
Settings._apply_external_env_overrides = lambda self: None
|
||||
|
||||
if not hasattr(context, "_cleanup_handlers"):
|
||||
context._cleanup_handlers = []
|
||||
|
||||
def cleanup():
|
||||
Settings._PROVIDER_ENV_MAP = original_map
|
||||
Settings._apply_external_env_overrides = original_override
|
||||
|
||||
context._cleanup_handlers.append(cleanup)
|
||||
|
||||
|
||||
@when("I check if any provider is configured")
|
||||
def step_check_any_provider(context):
|
||||
"""Check if any provider is configured."""
|
||||
@@ -372,3 +433,60 @@ def step_build_langsmith_config(context):
|
||||
def step_assert_langsmith_config_absent(context):
|
||||
"""Ensure LangSmith config is not generated when disabled."""
|
||||
assert context.langsmith_config is None
|
||||
|
||||
|
||||
@given("LangSmith is enabled without credentials")
|
||||
def step_enable_langsmith_without_credentials(context):
|
||||
"""Create a settings object with LangSmith enabled but missing key/project."""
|
||||
Settings._instance = None
|
||||
context.settings = Settings.model_construct(
|
||||
langsmith_enabled=True,
|
||||
langsmith_api_key=None,
|
||||
langsmith_project=None,
|
||||
)
|
||||
|
||||
|
||||
@when("I validate the LangSmith configuration")
|
||||
def step_validate_langsmith_configuration(context):
|
||||
"""Run validation against the current LangSmith settings."""
|
||||
assert hasattr(context, "settings"), "Settings must be initialized first."
|
||||
context.langsmith_validation = context.settings.validate_langsmith_configuration()
|
||||
|
||||
|
||||
@then("LangSmith validation should fail with {error_count:d} errors")
|
||||
def step_assert_langsmith_validation_errors(context, error_count):
|
||||
"""Assert the validation result and number of reported errors."""
|
||||
enabled, errors = context.langsmith_validation
|
||||
assert enabled is False
|
||||
assert len(errors) == error_count
|
||||
|
||||
|
||||
@given(
|
||||
'LangSmith is configured with API key "{api_key}", project "{project}", and endpoint "{endpoint}"'
|
||||
)
|
||||
def step_configure_langsmith_with_endpoint(context, api_key, project, endpoint):
|
||||
"""Construct LangSmith settings with explicit endpoint details."""
|
||||
Settings._instance = None
|
||||
context.settings = Settings.model_construct(
|
||||
langsmith_enabled=True,
|
||||
langsmith_api_key=api_key,
|
||||
langsmith_project=project,
|
||||
langsmith_endpoint=endpoint,
|
||||
)
|
||||
|
||||
|
||||
@when("I check if LangSmith is enabled")
|
||||
def step_check_langsmith_enabled(context):
|
||||
"""Evaluate whether LangSmith is enabled for the current settings."""
|
||||
assert hasattr(context, "settings"), "Settings must be initialized first."
|
||||
context.langsmith_enabled_flag = context.settings.is_langsmith_enabled
|
||||
|
||||
|
||||
@then('the "{env_var}" environment variable should equal "{expected}"')
|
||||
def step_verify_environment_variable(context, env_var, expected):
|
||||
"""Verify that an environment variable has the expected value."""
|
||||
assert os.environ.get(env_var) == expected
|
||||
if not hasattr(context, "env_vars_to_clean"):
|
||||
context.env_vars_to_clean = []
|
||||
if env_var not in context.env_vars_to_clean:
|
||||
context.env_vars_to_clean.append(env_var)
|
||||
|
||||
Reference in New Issue
Block a user