diff --git a/features/langchain_chat_provider_coverage.feature b/features/langchain_chat_provider_coverage.feature index b53ae518f..e9f784657 100644 --- a/features/langchain_chat_provider_coverage.feature +++ b/features/langchain_chat_provider_coverage.feature @@ -33,3 +33,37 @@ Feature: LangChain chat provider coverage When the provider generates changes and retries exhaust with nested errors Then the provider response should capture the nested retry failure And the progress callback should end at 100 percent even on failure + + @coverage @langchain @no_progress + Scenario: LangChain provider handles invalid state data without a progress callback + Given a LangChain chat provider is configured with a fake LangChain graph + When the provider generates changes without a progress callback and the graph returns invalid response data + Then the provider response should surface the state error without changes + And the progress callback should not receive updates + + @coverage @langchain @streaming @errors + Scenario: LangChain provider records partial streaming progress when the stream fails + Given a LangChain chat provider is configured with a fake LangChain graph + When the provider streams changes and the stream raises an exception after unknown node events + Then the provider response should capture the graph failure + And the progress callback should capture partial streaming progress before failure + + @coverage @langchain @retry + Scenario: LangChain provider reports failure when retry runner yields no attempts + Given a LangChain chat provider is configured with a fake LangChain graph + When the provider generates changes and the retry runner yields no attempts + Then the provider response should capture the graph failure + And the progress callback should end at 100 percent even on failure + + @coverage @langchain @retry + Scenario Outline: LangChain provider falls back to retry error strings when metadata is incomplete + Given a LangChain chat provider is configured with a fake LangChain graph + When the provider generates changes and the retry metadata is "" + Then the provider response should capture the nested retry failure + And the progress callback should end at 100 percent even on failure + + Examples: + | variant | + | missing-last-attempt | + | non-callable-exception | + | non-exception-result | diff --git a/features/settings_configuration.feature b/features/settings_configuration.feature index 7312cb825..fbbd560e8 100644 --- a/features/settings_configuration.feature +++ b/features/settings_configuration.feature @@ -58,3 +58,34 @@ Feature: Settings runtime helpers Scenario: LangSmith config builder returns None when disabled When I build the LangSmith config Then the LangSmith config should be absent + + Scenario: Relative vector store path resolves against cwd + Given the vector store path override is "relative/vector" + When I resolve the vector store path + Then the resolved vector store path should equal the override under the current working directory + + Scenario: LangSmith validation surfaces missing credentials + Given LangSmith is enabled without credentials + When I validate the LangSmith configuration + Then LangSmith validation should fail with 2 errors + + Scenario: LangSmith endpoint synchronization updates LangChain environment + Given LangSmith is configured with API key "abc123", project "coverage-runner", and endpoint "https://trace.local" + When I check if LangSmith is enabled + Then the "LANGCHAIN_ENDPOINT" environment variable should equal "https://trace.local" + + Scenario: Provider lookup trims api key suffix before resolving + Given the environment variable "OPENAI_API_KEY" is set to "env-key" + When I check if "openai_api_key" provider is configured + Then has_provider_configured should be True for "openai_api_key" + + Scenario: Provider lookup falls back to direct map entries + Given the environment variable "HF_TOKEN" is set to "hf-secret" + When I check if "hf_token" provider is configured + Then has_provider_configured should be True for "hf_token" + + Scenario: Custom provider candidate resolves env mapping + Given a temporary provider "custom" uses env "CUSTOM_API_KEY" + And the environment variable "CUSTOM_API_KEY" is set to "dynamic-key" + When I check if "custom_api_key" provider is configured using a constructed instance + Then has_provider_configured should be True for "custom_api_key" diff --git a/features/steps/langchain_chat_provider_steps.py b/features/steps/langchain_chat_provider_steps.py index 44931de24..60d56e981 100644 --- a/features/steps/langchain_chat_provider_steps.py +++ b/features/steps/langchain_chat_provider_steps.py @@ -99,6 +99,34 @@ def step_provider_generates_exception(context): context.failure_message = failure_message +@when( + "the provider generates changes without a progress callback and the graph returns invalid response data" +) +def step_provider_invalid_state_without_progress(context): + context.llm_instance.get_num_tokens = None + context.contexts = [MagicMock(spec=Context)] + context.contexts[0].content = "" + invalid_state = { + "generated_changes": "not-a-list", + "validation_result": "not-a-dict", + "error": "Graph returned invalid payload", + } + context.state_error_message = invalid_state["error"] + + with patch( + "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" + ) as graph_cls: + mock_graph = graph_cls.return_value + mock_graph.invoke.return_value = invalid_state + context.graph_instance = mock_graph + context.response = context.provider.generate_changes( + context.project, + context.plan, + context.contexts, + progress_callback=None, + ) + + @when("the provider streams changes with incremental graph events") def step_provider_streams_changes(context): context.provider = context.create_provider(supports_streaming=True) @@ -143,6 +171,36 @@ def step_provider_streams_changes(context): context.expected_token_count = 321 +@when( + "the provider streams changes and the stream raises an exception after unknown node events" +) +def step_provider_stream_failure_with_unknown_nodes(context): + context.provider = context.create_provider(supports_streaming=True) + failure_message = "Streaming interrupted by upstream failure" + streaming_events = [{"load_context": {}}, {"mystery_node": "unexpected-payload"}] + context.llm_instance.get_num_tokens.return_value = None + + def _failing_stream(): + for event in streaming_events: + yield event + raise RuntimeError(failure_message) + + with patch( + "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" + ) as graph_cls: + mock_graph = graph_cls.return_value + mock_graph.stream.return_value = _failing_stream() + context.graph_instance = mock_graph + context.response = context.provider.generate_changes( + context.project, + context.plan, + context.contexts, + progress_callback=context.progress_callback, + ) + + context.failure_message = failure_message + + @when("the provider generates changes and retries exhaust with nested errors") def step_provider_generates_retry_failure(context): final_message = "All retry attempts failed" @@ -174,6 +232,79 @@ def step_provider_generates_retry_failure(context): context.failure_message = final_message +@when('the provider generates changes and the retry metadata is "{variant}"') +def step_provider_incomplete_retry_metadata(context, variant): + class RetryError(Exception): + pass + + retry_error = RetryError(f"Incomplete retry metadata: {variant}") + + if variant == "missing-last-attempt": + retry_error.last_attempt = None + elif variant == "non-callable-exception": + + class _Attempt: + exception = "not-callable" + + retry_error.last_attempt = _Attempt() + elif variant == "non-exception-result": + + class _Attempt: + def exception(self): + return "not-exception" + + retry_error.last_attempt = _Attempt() + else: # pragma: no cover - defensive guard + raise AssertionError(f"Unknown retry metadata variant: {variant}") + + with patch( + "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" + ) as graph_cls: + mock_graph = graph_cls.return_value + mock_graph.invoke.side_effect = retry_error + context.graph_instance = mock_graph + context.response = context.provider.generate_changes( + context.project, + context.plan, + context.contexts, + progress_callback=context.progress_callback, + ) + + context.failure_message = str(retry_error) + + +@when("the provider generates changes and the retry runner yields no attempts") +def step_provider_retry_runner_without_attempts(context): + failure_message = "Retries exhausted while invoking plan generation graph" + + class _EmptyRetrying: + def __init__(self, *_, **__): + pass + + def __iter__(self): + return iter(()) + + with ( + patch( + "cleveragents.providers.llm.langchain_chat_provider.Retrying", + _EmptyRetrying, + ), + patch( + "cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph" + ) as graph_cls, + ): + mock_graph = graph_cls.return_value + context.graph_instance = mock_graph + context.response = context.provider.generate_changes( + context.project, + context.plan, + context.contexts, + progress_callback=context.progress_callback, + ) + + context.failure_message = failure_message + + @then( "the provider response should contain the generated change data and validation error" ) @@ -189,6 +320,19 @@ def step_assert_response_contains_validation_error(context): assert context.requested_models == ["test-model"] +@then("the provider response should surface the state error without changes") +def step_assert_state_error_without_changes(context): + assert context.response is not None + assert context.response.changes == [] + assert context.response.error_message == context.state_error_message + assert context.response.token_count == 0 + + +@then("the progress callback should not receive updates") +def step_assert_no_progress_updates(context): + assert context.progress_updates == [] + + @then("the progress callback should record the LangChain workflow milestones") def step_assert_progress_milestones(context): assert context.progress_updates == [5, 90, 100] @@ -226,6 +370,11 @@ def step_assert_streaming_progress(context): assert context.progress_updates == [5, 15, 40, 70, 90, 100] +@then("the progress callback should capture partial streaming progress before failure") +def step_assert_streaming_partial_progress(context): + assert context.progress_updates == [5, 15, 100, 100] + + @then("the LangChain graph stream should receive the thread-aware configuration") def step_assert_stream_call(context): call_args, call_kwargs = context.stream_call_args diff --git a/features/steps/settings_steps.py b/features/steps/settings_steps.py index 1f4ed3830..31bcbde4c 100644 --- a/features/steps/settings_steps.py +++ b/features/steps/settings_steps.py @@ -173,6 +173,33 @@ def step_check_storage_path(context, expected): assert str(context.storage_path) == expected +@given('the vector store path override is "{path_value}"') +def step_set_vector_store_override(context, path_value): + """Record the vector store path override for later resolution.""" + context.vector_store_override = path_value + + +@when("I resolve the vector store path") +def step_resolve_vector_store_path(context): + """Resolve the configured vector store path to an absolute location.""" + override = getattr(context, "vector_store_override", None) + assert override is not None, "Vector store path override must be provided first." + Settings._instance = None + settings = Settings.model_construct(vector_store_path=Path(override)) + context.resolved_vector_store_path = settings.resolve_vector_store_path() + + +@then( + "the resolved vector store path should equal the override under the current working directory" +) +def step_verify_resolved_vector_store_path(context): + """Ensure the vector store path is resolved relative to the current working directory.""" + override = getattr(context, "vector_store_override", None) + assert override is not None, "Vector store path override must be provided first." + expected = (Path.cwd() / Path(override)).resolve() + assert context.resolved_vector_store_path == expected + + @given('the environment is set to "{env}"') def step_set_environment(context, env): """Set the environment.""" @@ -246,6 +273,16 @@ def step_check_provider_configured(context, provider): ) +@when('I check if "{provider}" provider is configured using a constructed instance') +def step_check_provider_configured_constructed(context, provider): + """Check provider configuration using a lightweight constructed settings instance.""" + Settings._instance = None + settings = Settings.model_construct() + context.provider_configured = settings.has_provider_configured( + provider.lower().replace(" ", "") + ) + + @when('I clear the explicit "{provider}" key and recheck configuration') def step_clear_explicit_key_and_recheck(context, provider): """Clear the in-memory provider key and re-evaluate configuration.""" @@ -287,6 +324,30 @@ def step_clear_api_keys(context): os.environ.pop(key, None) +@given('a temporary provider "{provider}" uses env "{env_var}"') +def step_define_temporary_provider(context, provider, env_var): + """Temporarily register a provider mapping for provider lookup tests.""" + normalized = provider.lower().replace(" ", "_") + if not normalized.endswith("_api_key"): + normalized = f"{normalized}_api_key" + + original_map = Settings._PROVIDER_ENV_MAP + original_override = Settings._apply_external_env_overrides + + Settings._PROVIDER_ENV_MAP = dict(Settings._PROVIDER_ENV_MAP) + Settings._PROVIDER_ENV_MAP[normalized] = (env_var,) + Settings._apply_external_env_overrides = lambda self: None + + if not hasattr(context, "_cleanup_handlers"): + context._cleanup_handlers = [] + + def cleanup(): + Settings._PROVIDER_ENV_MAP = original_map + Settings._apply_external_env_overrides = original_override + + context._cleanup_handlers.append(cleanup) + + @when("I check if any provider is configured") def step_check_any_provider(context): """Check if any provider is configured.""" @@ -372,3 +433,60 @@ def step_build_langsmith_config(context): def step_assert_langsmith_config_absent(context): """Ensure LangSmith config is not generated when disabled.""" assert context.langsmith_config is None + + +@given("LangSmith is enabled without credentials") +def step_enable_langsmith_without_credentials(context): + """Create a settings object with LangSmith enabled but missing key/project.""" + Settings._instance = None + context.settings = Settings.model_construct( + langsmith_enabled=True, + langsmith_api_key=None, + langsmith_project=None, + ) + + +@when("I validate the LangSmith configuration") +def step_validate_langsmith_configuration(context): + """Run validation against the current LangSmith settings.""" + assert hasattr(context, "settings"), "Settings must be initialized first." + context.langsmith_validation = context.settings.validate_langsmith_configuration() + + +@then("LangSmith validation should fail with {error_count:d} errors") +def step_assert_langsmith_validation_errors(context, error_count): + """Assert the validation result and number of reported errors.""" + enabled, errors = context.langsmith_validation + assert enabled is False + assert len(errors) == error_count + + +@given( + 'LangSmith is configured with API key "{api_key}", project "{project}", and endpoint "{endpoint}"' +) +def step_configure_langsmith_with_endpoint(context, api_key, project, endpoint): + """Construct LangSmith settings with explicit endpoint details.""" + Settings._instance = None + context.settings = Settings.model_construct( + langsmith_enabled=True, + langsmith_api_key=api_key, + langsmith_project=project, + langsmith_endpoint=endpoint, + ) + + +@when("I check if LangSmith is enabled") +def step_check_langsmith_enabled(context): + """Evaluate whether LangSmith is enabled for the current settings.""" + assert hasattr(context, "settings"), "Settings must be initialized first." + context.langsmith_enabled_flag = context.settings.is_langsmith_enabled + + +@then('the "{env_var}" environment variable should equal "{expected}"') +def step_verify_environment_variable(context, env_var, expected): + """Verify that an environment variable has the expected value.""" + assert os.environ.get(env_var) == expected + if not hasattr(context, "env_vars_to_clean"): + context.env_vars_to_clean = [] + if env_var not in context.env_vars_to_clean: + context.env_vars_to_clean.append(env_var)