test(providers): add failing scenario for silent token-count exception swallowing (#10889)

features/steps/tdd_langchain_token_count_silent_failure_steps.py

@@ -0,0 +1,115 @@
+"""Step definitions for tdd_langchain_token_count_silent_failure.feature.
+
+This test captures bug #10395: LangChainChatProvider._estimate_token_usage()
+silently swallows exceptions from the LLM's ``get_num_tokens()`` call via a
+bare ``except Exception: return 0`` block.  When token counting raises any
+exception, the method returns 0 instead of propagating the error.  Downstream
+cost tracking then records zero tokens, producing inaccurate cost estimates
+with no visible error signal to the caller or operator.
+
+The scenario is tagged ``@tdd_expected_fail`` so the underlying assertion
+failure (confirming the bug exists) is inverted to a CI pass.  Once the
+fix for #10395 is merged, remove the ``@tdd_expected_fail`` tag from the
+feature file and this test will run normally as a regression guard.
+
+See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from behave import given, then, when
+from behave.runner import Context
+
+from cleveragents.domain.models.core import Context as AgentContext
+from cleveragents.domain.models.core import Plan
+from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider
+
+# The distinctive error message that must appear when the exception propagates.
+_TOKEN_COUNT_ERROR_MESSAGE = "simulated token count failure for TDD #10395"
+
+
+@given("a LangChainChatProvider configured with a mock LLM that raises on token count")
+def step_given_provider_with_raising_llm(context: Context) -> None:
+    """Create a LangChainChatProvider whose LLM raises on get_num_tokens()."""
+    context.raised_exception: Exception | None = None
+    context.returned_token_count: int | None = None
+
+    # Build a mock LLM whose get_num_tokens() always raises.
+    llm_instance = MagicMock()
+    llm_instance.get_num_tokens = MagicMock(
+        side_effect=RuntimeError(_TOKEN_COUNT_ERROR_MESSAGE)
+    )
+
+    def fake_llm_factory(model_id: str) -> MagicMock:
+        return llm_instance
+
+    context.llm_instance = llm_instance
+    context.provider = LangChainChatProvider(
+        name="tdd-test-provider",
+        model_id="tdd-test-model",
+        llm_factory=fake_llm_factory,
+        max_retries=1,
+        supports_streaming=False,
+    )
+
+    # Minimal plan and contexts for the token estimation call.
+    context.plan = MagicMock(spec=Plan)
+    context.plan.prompt = "TDD test prompt for issue #10395"
+    context.contexts = [MagicMock(spec=AgentContext)]
+    context.contexts[0].content = "TDD context content"
+
+
+@when("the provider attempts to count tokens for a message")
+def step_when_provider_counts_tokens(context: Context) -> None:
+    """Invoke _estimate_token_usage() directly and capture the outcome."""
+    try:
+        result = context.provider._estimate_token_usage(
+            context.llm_instance,
+            context.plan,
+            context.contexts,
+        )
+        # Bug is present: the exception was swallowed and 0 was returned.
+        context.returned_token_count = result
+        context.raised_exception = None
+    except Exception as exc:
+        # Fix is present: the exception propagated correctly.
+        context.raised_exception = exc
+        context.returned_token_count = None
+
+
+@then("a token count exception should be raised by the provider")
+def step_then_token_count_exception_raised(context: Context) -> None:
+    """Assert that the token counting exception propagated to the caller.
+
+    This assertion FAILS while the bug exists (the method returns 0 instead
+    of raising).  The ``@tdd_expected_fail`` tag inverts the result so CI
+    reports the scenario as passed until the fix is merged.
+
+    Uses ``AssertionError`` only — not ``ValueError`` or ``RuntimeError`` —
+    as required by CONTRIBUTING.md > TDD Issue Test Tags.
+    """
+    assert context.raised_exception is not None, (
+        f"Bug #10395: _estimate_token_usage() silently swallowed the exception "
+        f"from get_num_tokens() and returned {context.returned_token_count!r} "
+        "instead of propagating the error. "
+        "Expected an exception to be raised so that callers can detect the "
+        "failure and cost tracking does not record a zero-token entry."
+    )
+
+
+@then("the cost tracker should not record a zero-token usage entry")
+def step_then_no_zero_token_entry(context: Context) -> None:
+    """Assert that zero was not silently returned as the token count.
+
+    When the exception propagates correctly (fix present), this step
+    confirms that the caller received an exception rather than a misleading
+    zero-token count that would corrupt cost tracking records.
+    """
+    assert context.returned_token_count != 0, (
+        "Bug #10395: _estimate_token_usage() returned 0 tokens after "
+        "silently swallowing the exception from get_num_tokens(). "
+        "A zero-token return value causes downstream cost tracking to record "
+        "inaccurate usage data with no error signal to the operator."
+    )

features/tdd_langchain_token_count_silent_failure.feature

@@ -0,0 +1,28 @@
+@tdd_issue @tdd_issue_10395
+Feature: TDD Issue #10395 — Silent except Exception: return 0 in LangChainChatProvider._estimate_token_usage() causes inaccurate cost tracking
+  As a developer relying on accurate cost tracking
+  I want token counting failures in LangChainChatProvider._estimate_token_usage() to propagate
+  So that cost tracking never silently records zero tokens due to a suppressed exception
+
+  The `_estimate_token_usage()` method in
+  `cleveragents.providers.llm.langchain_chat_provider.LangChainChatProvider`
+  contains a bare `except Exception: return 0` block. When the LLM's
+  `get_num_tokens()` raises any exception, the method silently swallows it
+  and returns 0. Downstream cost tracking then records zero tokens, producing
+  inaccurate cost estimates with no visible error signal to the caller or
+  operator.
+
+  The scenario below is tagged `@tdd_expected_fail` because the assertion
+  FAILS while the bug exists. The tag inversion mechanism causes CI to report
+  the scenario as passed. Once the fix for #10395 is merged, the
+  `@tdd_expected_fail` tag must be removed so the scenario runs normally as
+  a regression guard.
+
+  See CONTRIBUTING.md > Bug Fix Workflow > TDD Issue Test Tags.
+
+  @tdd_expected_fail
+  Scenario: Token counting failure propagates as an error rather than returning zero
+    Given a LangChainChatProvider configured with a mock LLM that raises on token count
+    When the provider attempts to count tokens for a message
+    Then a token count exception should be raised by the provider
+    And the cost tracker should not record a zero-token usage entry