Feat: Added OpenAI and Antropic provider support and tests

This commit is contained in:
2025-12-08 23:02:29 -05:00
parent 896cc60d55
commit f82d9552d1
9 changed files with 379 additions and 50 deletions
+19
View File
@@ -0,0 +1,19 @@
Feature: Anthropic chat provider coverage
As a maintainer ensuring provider adapters stay validated
I want Behave scenarios for the Anthropic chat provider
So that the constructor guardrails and keyword handling stay covered
@unit @providers @anthropic
Scenario: Anthropic provider rejects missing API key
Given I have sample provider domain inputs
When I attempt to create an Anthropic chat provider without an API key
Then the Anthropic provider creation should fail with "Anthropic API key is required"
@unit @providers @anthropic
Scenario: Anthropic provider forwards keyword overrides to ChatAnthropic
Given I have sample provider domain inputs
When I create an Anthropic chat provider with API key "sk-anthropic-unit" and model "claude-3-5-sonnet-20241022" with custom overrides
And I request plan generation from the Anthropic provider
Then the Anthropic provider should construct ChatAnthropic with api key "sk-anthropic-unit" and model "claude-3-5-sonnet-20241022"
And the Anthropic provider should apply the custom overrides to ChatAnthropic
And the Anthropic provider metadata should report name "anthropic" and model "claude-3-5-sonnet-20241022"
+20 -2
View File
@@ -7,13 +7,31 @@ Feature: OpenAI chat provider coverage
Scenario: OpenAI provider instantiates ChatOpenAI with provided credentials
Given I have sample provider domain inputs
When I create an OpenAI chat provider with API key "sk-unit-test" and model "gpt-4o-mini"
And I request plan generation from the OpenAI provider
Then the OpenAI provider should construct ChatOpenAI with api key "sk-unit-test" and model "gpt-4o-mini"
And the OpenAI provider metadata should report name "openai" and model "gpt-4o-mini"
@unit @providers @openai
Scenario: OpenAI provider placeholder response returns no changes
Scenario: OpenAI provider stubbed response reports metadata
Given I have sample provider domain inputs
And I create an OpenAI chat provider with API key "sk-unit-test" and model "gpt-4o-mini"
When I request placeholder changes from the OpenAI provider
When I request plan generation from the OpenAI provider
Then the OpenAI provider response should contain no generated changes
And the OpenAI provider response should report the requested model without errors
@unit @providers @openai
Scenario: OpenAI provider rejects missing API key
Given I have sample provider domain inputs
When I attempt to create an OpenAI chat provider without an API key
Then the OpenAI provider creation should fail with error "OpenAI API key is required"
@unit @providers @openai
Scenario: OpenAI provider forwards optional organization and kwargs
Given I have sample provider domain inputs
And I set the OpenAI provider organization "org-test-123"
And I set the OpenAI provider extra kwargs "temperature=0.35,max_tokens=256"
When I create an OpenAI chat provider with API key "sk-unit-test" and model "gpt-4o-mini"
And I request plan generation from the OpenAI provider
Then the OpenAI provider should construct ChatOpenAI with api key "sk-unit-test" and model "gpt-4o-mini"
And the OpenAI provider should include organization "org-test-123" in the ChatOpenAI call
And the OpenAI provider should include kwargs "temperature=0.35,max_tokens=256" in the ChatOpenAI call
+129
View File
@@ -0,0 +1,129 @@
from __future__ import annotations
from unittest.mock import MagicMock, patch
from behave import then, when
from cleveragents.providers.llm.anthropic_provider import AnthropicChatProvider
def _register_cleanup(context, cleanup):
if hasattr(context, "add_cleanup"):
context.add_cleanup(cleanup)
else:
cleanup_handlers = getattr(context, "_cleanup_handlers", [])
cleanup_handlers.append(cleanup)
context._cleanup_handlers = cleanup_handlers
@when("I attempt to create an Anthropic chat provider without an API key")
def step_attempt_create_without_key(context):
context.anthropic_creation_error = None
try:
AnthropicChatProvider(api_key="", model="claude-3-opus-20240229")
except Exception as exc: # pragma: no cover - defensive test guard
context.anthropic_creation_error = exc
@then('the Anthropic provider creation should fail with "{error_message}"')
def step_assert_creation_failure(context, error_message):
error = getattr(context, "anthropic_creation_error", None)
assert error is not None, "Anthropic provider creation should have failed"
assert isinstance(error, ValueError)
assert str(error) == error_message
@when(
'I create an Anthropic chat provider with API key "{api_key}" and model "{model}" with custom overrides'
)
def step_create_anthropic_provider_with_overrides(context, api_key, model):
patcher = patch("cleveragents.providers.llm.anthropic_provider.ChatAnthropic")
context.chat_anthropic_patcher = patcher
mock_chat_class = patcher.start()
_register_cleanup(context, patcher.stop)
mock_chat_instance = MagicMock(name="ChatAnthropicInstance")
mock_chat_instance.get_num_tokens = MagicMock(return_value=0)
mock_chat_class.return_value = mock_chat_instance
context.anthropic_overrides = {
"temperature": 0.15,
"max_tokens": 2048,
"default_request_timeout": 12,
}
context.provider = AnthropicChatProvider(
api_key=api_key,
model=model,
max_retries=2,
**context.anthropic_overrides,
)
context.chat_anthropic_class = mock_chat_class
context.chat_anthropic_instance = mock_chat_instance
@when("I request plan generation from the Anthropic provider")
def step_request_plan_generation_anthropic(context):
patcher = patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
)
context.plan_generation_patcher = patcher
mock_graph_class = patcher.start()
_register_cleanup(context, patcher.stop)
mock_graph_instance = MagicMock(name="PlanGenerationGraphInstance")
mock_graph_instance.invoke.return_value = {
"generated_changes": [],
"validation_result": {"status": "PASS"},
"error": None,
}
mock_graph_instance.stream.return_value = iter(())
mock_graph_class.return_value = mock_graph_instance
context.plan_generation_graph = mock_graph_instance
context.plan_generation_graph_class = mock_graph_class
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
)
context.chat_anthropic_call = context.chat_anthropic_class.call_args
context.plan_generation_graph_call = mock_graph_class.call_args
@then(
'the Anthropic provider should construct ChatAnthropic with api key "{api_key}" and model "{model}"'
)
def step_assert_chat_anthropic_construction(context, api_key, model):
call = getattr(context, "chat_anthropic_call", None)
assert call is not None, "ChatAnthropic should have been constructed"
call_args, call_kwargs = call
assert call_args == (), "ChatAnthropic should be called with keyword arguments"
assert call_kwargs["api_key"] == api_key
assert call_kwargs["model"] == model
graph_call = getattr(context, "plan_generation_graph_call", None)
assert graph_call is not None, "PlanGenerationGraph should be constructed"
_, graph_kwargs = graph_call
assert graph_kwargs["llm"] is context.chat_anthropic_instance
@then("the Anthropic provider should apply the custom overrides to ChatAnthropic")
def step_assert_anthropic_overrides(context):
call = getattr(context, "chat_anthropic_call", None)
assert call is not None, "ChatAnthropic should have been constructed"
_, call_kwargs = call
overrides = getattr(context, "anthropic_overrides", None)
assert overrides, "Custom overrides should have been captured"
for key, value in overrides.items():
assert call_kwargs.get(key) == value, f"Override {key} was not forwarded"
@then(
'the Anthropic provider metadata should report name "{expected_name}" and model "{expected_model}"'
)
def step_assert_anthropic_metadata(context, expected_name, expected_model):
provider = getattr(context, "provider", None)
assert provider is not None, "Anthropic provider should exist in context"
assert provider.name == expected_name
assert provider.model_id == expected_model
+118 -6
View File
@@ -1,5 +1,7 @@
from __future__ import annotations
import ast
from typing import Any
from unittest.mock import MagicMock, patch
from behave import given, then, when
@@ -17,6 +19,28 @@ def _register_cleanup(context, cleanup):
context._cleanup_handlers = cleanup_handlers
def _parse_kwargs_string(kwargs_string: str) -> dict[str, Any]:
if not kwargs_string:
return {}
result: dict[str, Any] = {}
for entry in kwargs_string.split(","):
entry = entry.strip()
if not entry:
continue
if "=" not in entry:
result[entry] = True
continue
key, value = entry.split("=", 1)
key = key.strip()
value = value.strip()
try:
parsed_value = ast.literal_eval(value)
except Exception:
parsed_value = value
result[key] = parsed_value
return result
@given("I have sample provider domain inputs")
def step_sample_provider_inputs(context):
context.project = MagicMock(spec=Project)
@@ -26,6 +50,19 @@ def step_sample_provider_inputs(context):
context.contexts[0].content = "Sample context entry"
@given('I set the OpenAI provider organization "{organization}"')
def step_set_openai_provider_org(context, organization):
if organization.lower() == "none":
context.openai_provider_org = None
else:
context.openai_provider_org = organization
@given('I set the OpenAI provider extra kwargs "{kwargs_string}"')
def step_set_openai_provider_kwargs(context, kwargs_string):
context.openai_provider_kwargs = _parse_kwargs_string(kwargs_string)
@given(
'I create an OpenAI chat provider with API key "{api_key}" and model "{model_id}"'
)
@@ -38,21 +75,57 @@ def step_create_openai_provider(context, api_key, model_id):
mock_chat_openai_class = patcher.start()
_register_cleanup(context, patcher.stop)
mock_chat_openai_instance = MagicMock(name="ChatOpenAIInstance")
mock_chat_openai_instance.get_num_tokens = MagicMock(return_value=0)
mock_chat_openai_class.return_value = mock_chat_openai_instance
context.provider = OpenAIChatProvider(api_key=api_key, model=model_id)
context.chat_openai_call = mock_chat_openai_class.call_args
organization = getattr(context, "openai_provider_org", None)
extra_kwargs = dict(getattr(context, "openai_provider_kwargs", {}))
context.provider = OpenAIChatProvider(
api_key=api_key,
model=model_id,
organization=organization,
**extra_kwargs,
)
context.chat_openai_class = mock_chat_openai_class
context.chat_openai_instance = mock_chat_openai_instance
context.requested_credentials = {"api_key": api_key, "model": model_id}
@when("I request placeholder changes from the OpenAI provider")
def step_request_placeholder_changes(context):
@when("I attempt to create an OpenAI chat provider without an API key")
def step_openai_provider_without_api_key(context):
context.openai_provider_error = None
try:
OpenAIChatProvider(api_key="", model="gpt-4o-mini")
except Exception as exc: # pragma: no cover - defensive logging only
context.openai_provider_error = exc
@when("I request plan generation from the OpenAI provider")
def step_request_plan_generation(context):
patcher = patch(
"cleveragents.providers.llm.langchain_chat_provider.PlanGenerationGraph"
)
context.plan_generation_patcher = patcher
mock_graph_class = patcher.start()
_register_cleanup(context, patcher.stop)
mock_graph_instance = MagicMock(name="PlanGenerationGraphInstance")
mock_graph_instance.invoke.return_value = {
"generated_changes": [],
"validation_result": {"status": "PASS"},
"error": None,
}
mock_graph_instance.stream.return_value = iter(())
mock_graph_class.return_value = mock_graph_instance
context.plan_generation_graph = mock_graph_instance
context.plan_generation_graph_class = mock_graph_class
context.response = context.provider.generate_changes(
context.project,
context.plan,
context.contexts,
)
context.chat_openai_call = context.chat_openai_class.call_args
context.plan_generation_graph_call = mock_graph_class.call_args
@then(
@@ -64,7 +137,38 @@ def step_assert_chat_openai_constructor(context, api_key, model_id):
assert call_args == (), "ChatOpenAI should be called with keyword arguments"
assert call_kwargs["api_key"] == api_key
assert call_kwargs["model"] == model_id
assert context.provider.llm is context.chat_openai_instance
assert context.plan_generation_graph_call is not None, (
"PlanGenerationGraph should receive the ChatOpenAI instance"
)
_, graph_kwargs = context.plan_generation_graph_call
assert graph_kwargs["llm"] is context.chat_openai_instance
@then(
'the OpenAI provider should include organization "{expected_organization}" in the ChatOpenAI call'
)
def step_assert_chat_openai_org(context, expected_organization):
assert context.chat_openai_call is not None, "ChatOpenAI should have been called"
_, call_kwargs = context.chat_openai_call
normalized = (
None if expected_organization.lower() == "none" else expected_organization
)
assert call_kwargs.get("organization") == normalized
@then(
'the OpenAI provider should include kwargs "{kwargs_string}" in the ChatOpenAI call'
)
def step_assert_chat_openai_kwargs(context, kwargs_string):
assert context.chat_openai_call is not None, "ChatOpenAI should have been called"
_, call_kwargs = context.chat_openai_call
expected_kwargs = _parse_kwargs_string(kwargs_string)
for key, value in expected_kwargs.items():
assert key in call_kwargs, f"Expected {key} in ChatOpenAI kwargs"
assert call_kwargs[key] == value, (
f"Expected ChatOpenAI kwargs[{key!r}] to equal {value!r}"
)
@then(
@@ -87,3 +191,11 @@ def step_assert_placeholder_metadata(context):
assert context.response is not None
assert context.response.model_used == context.provider.model_id
assert context.response.error_message in (None, "")
@then('the OpenAI provider creation should fail with error "{message}"')
def step_assert_openai_provider_error(context, message):
error = getattr(context, "openai_provider_error", None)
assert error is not None, "Expected the provider to raise an error"
assert isinstance(error, ValueError)
assert str(error) == message
+15 -4
View File
@@ -1672,6 +1672,16 @@ Notes: Record provider-specific nuances, API limitations, and testing fixtures.
- `features/openai_provider.feature:7-22` now runs end-to-end against the real provider adapter (with LangChain mocked), giving regression coverage for constructor wiring and placeholder outputs until real streaming paths land.
- Executed `behave features/openai_provider.feature` locally; both scenarios pass and assert no generated changes while reporting the requested model, preventing regressions in provider bootstrap logic.
**2025-12-09: OpenAI provider LangGraph stub coverage refresh**
- Updated `features/openai_provider.feature:7-31` to exercise the `LangChainChatProvider.generate_changes` path so ChatOpenAI instantiation and LangGraph invocation occur under test.
- Step definitions in `features/steps/openai_provider_steps.py:17-191` now patch `PlanGenerationGraph` directly, assert organization/kwargs forwarding, and guarantee deterministic responses for CI coverage.
- Re-ran `behave features/openai_provider.feature` to document the green run for this refreshed coverage and keep the LangGraph/provider integration verified in CI logs.
**2025-12-09: Anthropic provider adapter + coverage**
- Added `src/cleveragents/providers/llm/anthropic_provider.py:1-36` implementing `AnthropicChatProvider`, a LangChain-backed adapter that validates API keys, forwards override kwargs, and streams progress through `LangChainChatProvider`.
- Authored `features/anthropic_provider.feature:1-19` with matching steps in `features/steps/anthropic_provider_steps.py:1-129` to cover missing-key errors, constructor overrides, and metadata reporting while asserting `PlanGenerationGraph` receives the mocked `ChatAnthropic` instance.
- Executed `behave features/anthropic_provider.feature` locally to confirm both Anthropic scenarios pass alongside the refreshed OpenAI suite.
**Additional LangChain/LangGraph Tasks for Phase 4:**
- Implement `CodeGenerationGraph` using LangGraph for multi-step code generation
- Create `AutoDebugGraph` with conditional edges for error detection and retry
@@ -4173,12 +4183,12 @@ If you can do all of the above by end of Day 1, you're on track!
- [ ] Code: Provider registry and runtime integration (Phase 4 scope)
- [x] Implement `ProviderRegistry` under `src/cleveragents/providers/registry.py` that discovers configured providers from `Settings`, reports capability metadata, and selects defaults based on `CLEVERAGENTS_DEFAULT_PROVIDER` / `CLEVERAGENTS_DEFAULT_MODEL`.
- [ ] Create concrete provider adapters in `src/cleveragents/providers/llm/` (OpenAI, Anthropic, Google, OpenRouter at minimum) wrapping LangChain chat models behind `AIProviderInterface`, including streaming hooks, retry annotations, and token accounting.
- [X] Implement `OpenAIChatProvider` placeholder in `src/cleveragents/providers/llm/openai_provider.py:1` wrapping `ChatOpenAI` with API key/model inputs and returning metadata-only responses until real streaming lands. Behave coverage lives in `features/openai_provider.feature:7-22` with steps under `features/steps/openai_provider_steps.py:20-89`.
- [ ] Replace placeholder `generate_changes()` with a real LangChain-powered runnable that invokes `ChatOpenAI` (or the selected provider) to produce structured plan deltas, maps them into `ProviderResponse.changes`, and surfaces model tokens/errors via LangSmith metadata.
- [X] Implement `OpenAIChatProvider` in `src/cleveragents/providers/llm/openai_provider.py:1-60` as a LangChain-backed adapter that validates API keys, forwards organization/kwargs overrides, and delegates execution to `LangChainChatProvider`.
- [X] Replace the placeholder `generate_changes()` with the real LangChain-powered workflow by invoking `LangChainChatProvider.generate_changes`, ensuring PlanGenerationGraph produces structured plan deltas and token estimates.
- [ ] Add streaming support that forwards `ChatOpenAI.stream()` events to PlanService/CLI consumers, including progress callbacks and graceful cancellation semantics.
- [ ] Capture token usage + cost metrics from the `LLMResult` and persist them on `ProviderResponse.token_count`, logging through structlog with LangSmith tags.
- [ ] Expand Behave + Robot coverage to exercise real OpenAI provider flows (using patched clients) covering success, streaming, error, and retry scenarios.
- [ ] Implement Anthropic chat provider adapter (Claude) with parity tests.
- [X] Implement Anthropic chat provider adapter (Claude) with parity tests (`src/cleveragents/providers/llm/anthropic_provider.py:1-36`).
- [ ] Implement Google Gemini provider adapter with parity tests.
- [ ] Implement OpenRouter provider adapter with org header support.
- [ ] Extend `Settings` with provider/model defaults, per-provider configuration (Azure deployments, OpenRouter org, etc.), and validation so missing env vars generate actionable errors.
@@ -4193,7 +4203,8 @@ If you can do all of the above by end of Day 1, you're on track!
- [ ] Extend ADR-008 (or add an addendum) capturing the registry implementation details, override order (CLI → env → defaults), and how new providers plug into the system.
- [ ] Tests: Provider selection and CLI override coverage
- [x] Create a dedicated Behave suite (`features/provider_registry_coverage.feature` + `features/steps/provider_registry_steps.py`) that simulates different env configurations, verifies registry discovery, default selection, CLI overrides, and failure messaging.
- [X] Add targeted OpenAI provider placeholder coverage in `features/openai_provider.feature:7-22` with steps `features/steps/openai_provider_steps.py:20-89` to assert ChatOpenAI constructor wiring and metadata-only responses until full provider integration completes.
- [X] Add LangGraph-backed OpenAI provider coverage in `features/openai_provider.feature:7-31` with steps `features/steps/openai_provider_steps.py:17-191` to assert constructor wiring, kwargs/organization overrides, missing-key errors, and PlanGenerationGraph invocation.
- [X] Add Anthropic provider coverage in `features/anthropic_provider.feature:1-19` with steps `features/steps/anthropic_provider_steps.py:1-129` validating API key enforcement, override forwarding, and metadata reporting.
- [X] Add Behave scenarios to `features/plan_service.feature` ensuring `plan build` succeeds when provider keys are set, fails with actionable guidance when they are missing, and honors `--provider/--model` flags. (COMPLETED 2025-12-08)
- [ ] Add Robot integration tests (e.g., `robot/provider_registry.robot`) that exercise real CLI flows with env overrides, confirm DI wiring returns the expected provider type, and capture logs showing selected providers.
- [ ] Update existing streaming/auto-debug Behave suites to cover mixed-provider scenarios (e.g., build with OpenAI, auto-debug fallback to Anthropic) and ensure token accounting + metadata propagate into LangGraph state.
+8 -2
View File
@@ -373,14 +373,20 @@ def tell(
str | None,
typer.Option(
"--provider",
help="Override the default AI provider (takes precedence over CLEVERAGENTS_DEFAULT_PROVIDER)",
help=(
"Override the default AI provider "
"(takes precedence over CLEVERAGENTS_DEFAULT_PROVIDER)"
),
),
] = None,
model: Annotated[
str | None,
typer.Option(
"--model",
help="Override the default model (takes precedence over CLEVERAGENTS_DEFAULT_MODEL)",
help=(
"Override the default model "
"(takes precedence over CLEVERAGENTS_DEFAULT_MODEL)"
),
),
] = None,
stream: Annotated[
@@ -0,0 +1,36 @@
from __future__ import annotations
from typing import Any
from langchain_anthropic import ChatAnthropic
from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider
class AnthropicChatProvider(LangChainChatProvider):
"""Concrete provider adapter for Anthropic Claude models."""
def __init__(
self,
*,
api_key: str,
model: str = "claude-sonnet-4-20250514",
max_retries: int = 3,
**llm_kwargs: Any,
) -> None:
if not api_key:
raise ValueError("Anthropic API key is required")
def factory(resolved_model: str) -> ChatAnthropic:
kwargs: dict[str, Any] = {"model": resolved_model, "api_key": api_key}
if llm_kwargs:
kwargs.update(llm_kwargs)
return ChatAnthropic(**kwargs)
super().__init__(
name="anthropic",
model_id=model,
llm_factory=factory,
max_retries=max_retries,
supports_streaming=True,
)
@@ -1,42 +1,39 @@
from typing import Callable
from cleveragents.domain.models.core import Context, Plan, Project
from cleveragents.domain.providers.ai_provider import (
AIProviderInterface,
ProviderResponse,
)
from __future__ import annotations
from typing import Any
from langchain_openai import ChatOpenAI
from cleveragents.providers.llm.langchain_chat_provider import LangChainChatProvider
class OpenAIChatProvider(AIProviderInterface):
"""
OpenAI-specific chat provider.
"""
def __init__(self, api_key: str, model: str = "gpt-4o"):
self._api_key = api_key
self._model_id = model
self.llm = ChatOpenAI(api_key=api_key, model=model)
class OpenAIChatProvider(LangChainChatProvider):
"""Concrete LangChain-backed provider for OpenAI chat models."""
@property
def name(self) -> str:
return "openai"
@property
def model_id(self) -> str:
return self._model_id
def generate_changes(
def __init__(
self,
project: Project,
plan: Plan,
contexts: list[Context],
progress_callback: Callable[[int], None] | None = None,
) -> ProviderResponse:
# This is a placeholder. The actual implementation will be added in a future commit.
# For now, we return an empty response.
return ProviderResponse(
changes=[],
model_used=self.model_id,
token_count=0,
error_message=None,
*,
api_key: str,
model: str = "gpt-4o",
max_retries: int = 3,
organization: str | None = None,
**llm_kwargs: Any,
) -> None:
if not api_key:
raise ValueError("OpenAI API key is required")
def factory(resolved_model: str) -> ChatOpenAI:
kwargs: dict[str, Any] = {"api_key": api_key, "model": resolved_model}
if organization:
kwargs["organization"] = organization
if llm_kwargs:
kwargs.update(llm_kwargs)
return ChatOpenAI(**kwargs)
super().__init__(
name="openai",
model_id=model,
llm_factory=factory,
max_retries=max_retries,
supports_streaming=True,
)
+2 -1
View File
@@ -465,7 +465,8 @@ class ProviderRegistry:
azure_endpoint = endpoint_override or self._settings.azure_openai_endpoint
if not azure_endpoint:
raise ValueError(
"Azure OpenAI endpoint not configured. Set AZURE_OPENAI_ENDPOINT or CLEVERAGENTS_AZURE_OPENAI_ENDPOINT."
"Azure OpenAI endpoint not configured. "
"Set AZURE_OPENAI_ENDPOINT or CLEVERAGENTS_AZURE_OPENAI_ENDPOINT."
)
api_version_override = _coerce_optional_str(