diff --git a/features/steps/tdd_langchain_token_count_silent_failure_steps.py b/features/steps/tdd_langchain_token_count_silent_failure_steps.py new file mode 100644 index 000000000..78ba1411c --- /dev/null +++ b/features/steps/tdd_langchain_token_count_silent_failure_steps.py @@ -0,0 +1,115 @@ +"""Step definitions for tdd_langchain_token_count_silent_failure.feature. + +This test captures bug #10395: LangChainChatProvider._estimate_token_usage() +silently swallows exceptions from the LLM's ``get_num_tokens()`` call via a +bare ``except Exception: return 0`` block. When token counting raises any +exception, the method returns 0 instead of propagating the error. Downstream +cost tracking then records zero tokens, producing inaccurate cost estimates +with no visible error signal to the caller or operator. + +The scenario is tagged ``@tdd_expected_fail`` so the underlying assertion +failure (confirming the bug exists) is inverted to a CI pass. Once the +fix for #10395 is merged, remove the ``@tdd_expected_fail`` tag from the +feature file and this test will run normally as a regression guard. + +See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.domain.models.core import Context as AgentContext +from cleveragents.domain.models.core import Plan +from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider + +# The distinctive error message that must appear when the exception propagates. +_TOKEN_COUNT_ERROR_MESSAGE = "simulated token count failure for TDD #10395" + + +@given("a LangChainChatProvider configured with a mock LLM that raises on token count") +def step_given_provider_with_raising_llm(context: Context) -> None: + """Create a LangChainChatProvider whose LLM raises on get_num_tokens().""" + context.raised_exception: Exception | None = None + context.returned_token_count: int | None = None + + # Build a mock LLM whose get_num_tokens() always raises. + llm_instance = MagicMock() + llm_instance.get_num_tokens = MagicMock( + side_effect=RuntimeError(_TOKEN_COUNT_ERROR_MESSAGE) + ) + + def fake_llm_factory(model_id: str) -> MagicMock: + return llm_instance + + context.llm_instance = llm_instance + context.provider = LangChainChatProvider( + name="tdd-test-provider", + model_id="tdd-test-model", + llm_factory=fake_llm_factory, + max_retries=1, + supports_streaming=False, + ) + + # Minimal plan and contexts for the token estimation call. + context.plan = MagicMock(spec=Plan) + context.plan.prompt = "TDD test prompt for issue #10395" + context.contexts = [MagicMock(spec=AgentContext)] + context.contexts[0].content = "TDD context content" + + +@when("the provider attempts to count tokens for a message") +def step_when_provider_counts_tokens(context: Context) -> None: + """Invoke _estimate_token_usage() directly and capture the outcome.""" + try: + result = context.provider._estimate_token_usage( + context.llm_instance, + context.plan, + context.contexts, + ) + # Bug is present: the exception was swallowed and 0 was returned. + context.returned_token_count = result + context.raised_exception = None + except Exception as exc: + # Fix is present: the exception propagated correctly. + context.raised_exception = exc + context.returned_token_count = None + + +@then("a token count exception should be raised by the provider") +def step_then_token_count_exception_raised(context: Context) -> None: + """Assert that the token counting exception propagated to the caller. + + This assertion FAILS while the bug exists (the method returns 0 instead + of raising). The ``@tdd_expected_fail`` tag inverts the result so CI + reports the scenario as passed until the fix is merged. + + Uses ``AssertionError`` only — not ``ValueError`` or ``RuntimeError`` — + as required by CONTRIBUTING.md > TDD Issue Test Tags. + """ + assert context.raised_exception is not None, ( + f"Bug #10395: _estimate_token_usage() silently swallowed the exception " + f"from get_num_tokens() and returned {context.returned_token_count!r} " + "instead of propagating the error. " + "Expected an exception to be raised so that callers can detect the " + "failure and cost tracking does not record a zero-token entry." + ) + + +@then("the cost tracker should not record a zero-token usage entry") +def step_then_no_zero_token_entry(context: Context) -> None: + """Assert that zero was not silently returned as the token count. + + When the exception propagates correctly (fix present), this step + confirms that the caller received an exception rather than a misleading + zero-token count that would corrupt cost tracking records. + """ + assert context.returned_token_count != 0, ( + "Bug #10395: _estimate_token_usage() returned 0 tokens after " + "silently swallowing the exception from get_num_tokens(). " + "A zero-token return value causes downstream cost tracking to record " + "inaccurate usage data with no error signal to the operator." + ) diff --git a/features/tdd_langchain_token_count_silent_failure.feature b/features/tdd_langchain_token_count_silent_failure.feature new file mode 100644 index 000000000..7e5c8c90f --- /dev/null +++ b/features/tdd_langchain_token_count_silent_failure.feature @@ -0,0 +1,28 @@ +@tdd_issue @tdd_issue_10395 +Feature: TDD Issue #10395 — Silent except Exception: return 0 in LangChainChatProvider._estimate_token_usage() causes inaccurate cost tracking + As a developer relying on accurate cost tracking + I want token counting failures in LangChainChatProvider._estimate_token_usage() to propagate + So that cost tracking never silently records zero tokens due to a suppressed exception + + The `_estimate_token_usage()` method in + `cleveragents.providers.llm.langchain_chat_provider.LangChainChatProvider` + contains a bare `except Exception: return 0` block. When the LLM's + `get_num_tokens()` raises any exception, the method silently swallows it + and returns 0. Downstream cost tracking then records zero tokens, producing + inaccurate cost estimates with no visible error signal to the caller or + operator. + + The scenario below is tagged `@tdd_expected_fail` because the assertion + FAILS while the bug exists. The tag inversion mechanism causes CI to report + the scenario as passed. Once the fix for #10395 is merged, the + `@tdd_expected_fail` tag must be removed so the scenario runs normally as + a regression guard. + + See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags. + + @tdd_expected_fail + Scenario: Token counting failure propagates as an error rather than returning zero + Given a LangChainChatProvider configured with a mock LLM that raises on token count + When the provider attempts to count tokens for a message + Then a token count exception should be raised by the provider + And the cost tracker should not record a zero-token usage entry