From a4aeea3e7c41b2039e2f3395e49d6f97c65c06ee Mon Sep 17 00:00:00 2001 From: HAL9000 Date: Tue, 28 Apr 2026 07:52:02 +0000 Subject: [PATCH] test(providers): add failing scenario for silent token-count exception swallowing Adds a TDD issue-capture test for bug #10395. The _estimate_token_usage() method in LangChainChatProvider contains a bare except Exception: return 0 block that silently swallows any exception raised by the LLM's get_num_tokens() call, returning 0 instead of propagating the error. Downstream cost tracking then records zero tokens, producing inaccurate cost estimates with no visible error signal to the caller or operator. The new Behave scenario in features/tdd_langchain_token_count_silent_failure.feature is tagged @tdd_issue, @tdd_issue_10395, and @tdd_expected_fail. The assertion fails against the current unfixed code (confirming the bug exists), and the @tdd_expected_fail tag inverts the result so CI reports the scenario as passed until the fix is merged. ISSUES CLOSED: #10395 --- ...gchain_token_count_silent_failure_steps.py | 115 ++++++++++++++++++ ...ngchain_token_count_silent_failure.feature | 28 +++++ 2 files changed, 143 insertions(+) create mode 100644 features/steps/tdd_langchain_token_count_silent_failure_steps.py create mode 100644 features/tdd_langchain_token_count_silent_failure.feature diff --git a/features/steps/tdd_langchain_token_count_silent_failure_steps.py b/features/steps/tdd_langchain_token_count_silent_failure_steps.py new file mode 100644 index 000000000..78ba1411c --- /dev/null +++ b/features/steps/tdd_langchain_token_count_silent_failure_steps.py @@ -0,0 +1,115 @@ +"""Step definitions for tdd_langchain_token_count_silent_failure.feature. + +This test captures bug #10395: LangChainChatProvider._estimate_token_usage() +silently swallows exceptions from the LLM's ``get_num_tokens()`` call via a +bare ``except Exception: return 0`` block. When token counting raises any +exception, the method returns 0 instead of propagating the error. Downstream +cost tracking then records zero tokens, producing inaccurate cost estimates +with no visible error signal to the caller or operator. + +The scenario is tagged ``@tdd_expected_fail`` so the underlying assertion +failure (confirming the bug exists) is inverted to a CI pass. Once the +fix for #10395 is merged, remove the ``@tdd_expected_fail`` tag from the +feature file and this test will run normally as a regression guard. + +See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from behave import given, then, when +from behave.runner import Context + +from cleveragents.domain.models.core import Context as AgentContext +from cleveragents.domain.models.core import Plan +from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider + +# The distinctive error message that must appear when the exception propagates. +_TOKEN_COUNT_ERROR_MESSAGE = "simulated token count failure for TDD #10395" + + +@given("a LangChainChatProvider configured with a mock LLM that raises on token count") +def step_given_provider_with_raising_llm(context: Context) -> None: + """Create a LangChainChatProvider whose LLM raises on get_num_tokens().""" + context.raised_exception: Exception | None = None + context.returned_token_count: int | None = None + + # Build a mock LLM whose get_num_tokens() always raises. + llm_instance = MagicMock() + llm_instance.get_num_tokens = MagicMock( + side_effect=RuntimeError(_TOKEN_COUNT_ERROR_MESSAGE) + ) + + def fake_llm_factory(model_id: str) -> MagicMock: + return llm_instance + + context.llm_instance = llm_instance + context.provider = LangChainChatProvider( + name="tdd-test-provider", + model_id="tdd-test-model", + llm_factory=fake_llm_factory, + max_retries=1, + supports_streaming=False, + ) + + # Minimal plan and contexts for the token estimation call. + context.plan = MagicMock(spec=Plan) + context.plan.prompt = "TDD test prompt for issue #10395" + context.contexts = [MagicMock(spec=AgentContext)] + context.contexts[0].content = "TDD context content" + + +@when("the provider attempts to count tokens for a message") +def step_when_provider_counts_tokens(context: Context) -> None: + """Invoke _estimate_token_usage() directly and capture the outcome.""" + try: + result = context.provider._estimate_token_usage( + context.llm_instance, + context.plan, + context.contexts, + ) + # Bug is present: the exception was swallowed and 0 was returned. + context.returned_token_count = result + context.raised_exception = None + except Exception as exc: + # Fix is present: the exception propagated correctly. + context.raised_exception = exc + context.returned_token_count = None + + +@then("a token count exception should be raised by the provider") +def step_then_token_count_exception_raised(context: Context) -> None: + """Assert that the token counting exception propagated to the caller. + + This assertion FAILS while the bug exists (the method returns 0 instead + of raising). The ``@tdd_expected_fail`` tag inverts the result so CI + reports the scenario as passed until the fix is merged. + + Uses ``AssertionError`` only — not ``ValueError`` or ``RuntimeError`` — + as required by CONTRIBUTING.md > TDD Issue Test Tags. + """ + assert context.raised_exception is not None, ( + f"Bug #10395: _estimate_token_usage() silently swallowed the exception " + f"from get_num_tokens() and returned {context.returned_token_count!r} " + "instead of propagating the error. " + "Expected an exception to be raised so that callers can detect the " + "failure and cost tracking does not record a zero-token entry." + ) + + +@then("the cost tracker should not record a zero-token usage entry") +def step_then_no_zero_token_entry(context: Context) -> None: + """Assert that zero was not silently returned as the token count. + + When the exception propagates correctly (fix present), this step + confirms that the caller received an exception rather than a misleading + zero-token count that would corrupt cost tracking records. + """ + assert context.returned_token_count != 0, ( + "Bug #10395: _estimate_token_usage() returned 0 tokens after " + "silently swallowing the exception from get_num_tokens(). " + "A zero-token return value causes downstream cost tracking to record " + "inaccurate usage data with no error signal to the operator." + ) diff --git a/features/tdd_langchain_token_count_silent_failure.feature b/features/tdd_langchain_token_count_silent_failure.feature new file mode 100644 index 000000000..7e5c8c90f --- /dev/null +++ b/features/tdd_langchain_token_count_silent_failure.feature @@ -0,0 +1,28 @@ +@tdd_issue @tdd_issue_10395 +Feature: TDD Issue #10395 — Silent except Exception: return 0 in LangChainChatProvider._estimate_token_usage() causes inaccurate cost tracking + As a developer relying on accurate cost tracking + I want token counting failures in LangChainChatProvider._estimate_token_usage() to propagate + So that cost tracking never silently records zero tokens due to a suppressed exception + + The `_estimate_token_usage()` method in + `cleveragents.providers.llm.langchain_chat_provider.LangChainChatProvider` + contains a bare `except Exception: return 0` block. When the LLM's + `get_num_tokens()` raises any exception, the method silently swallows it + and returns 0. Downstream cost tracking then records zero tokens, producing + inaccurate cost estimates with no visible error signal to the caller or + operator. + + The scenario below is tagged `@tdd_expected_fail` because the assertion + FAILS while the bug exists. The tag inversion mechanism causes CI to report + the scenario as passed. Once the fix for #10395 is merged, the + `@tdd_expected_fail` tag must be removed so the scenario runs normally as + a regression guard. + + See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags. + + @tdd_expected_fail + Scenario: Token counting failure propagates as an error rather than returning zero + Given a LangChainChatProvider configured with a mock LLM that raises on token count + When the provider attempts to count tokens for a message + Then a token count exception should be raised by the provider + And the cost tracker should not record a zero-token usage entry -- 2.52.0