fix(agents/graphs/plan_generation): _validate always passes for code longer than 10 characters, making LLM validation ineffective #10746

Closed
HAL9000 wants to merge 5 commits from fix/10480-validate-logic-error into master
5 changed files with 9 additions and 9 deletions
+3 -3
View File
@@ -18,6 +18,6 @@ Feature: A2A Python SDK is a declared project dependency
Then the import should succeed without errors
@tdd_issue @tdd_issue_4273
Scenario: a2a SDK provides the A2AClient class
When I import "a2a.client" and access "A2AClient"
Then the "A2AClient" class should be available
Scenario: a2a SDK provides the Client class
When I import "a2a.client" and access "Client"
Then the "Client" class should be available
+2 -2
View File
@@ -375,7 +375,7 @@ Workflow Invoke Method Returns Complete State
... from cleveragents.agents.plan_generation import PlanGenerationGraph
... from langchain_community.llms import FakeListLLM
... from cleveragents.domain.models.core import Project, Plan, Context
... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3))
... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['PASS: analysis complete']*10))
... project = Project(id=1, name='test_project', path=Path('/tmp/test_project'))
... plan = Plan(id=1, project_id=1, name='Logging Plan', prompt='Add logging')
... contexts = [Context(plan_id=plan.id, path='app.py', content='def main(): pass')]
@@ -405,7 +405,7 @@ Workflow Stream Method Yields Events
... from cleveragents.agents.plan_generation import PlanGenerationGraph
... from langchain_community.llms import FakeListLLM
... from cleveragents.domain.models.core import Project, Plan, Context
... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['test']*3))
... graph = PlanGenerationGraph(llm=FakeListLLM(responses=['PASS: analysis complete']*10))
... project = Project(id=1, name='test_project', path=Path('/tmp/test_project'))
... plan = Plan(id=1, project_id=1, name='Feature Plan', prompt='Add feature')
... contexts = [Context(plan_id=plan.id, path='app.py', content='# app')]
@@ -527,8 +527,8 @@ class PlanGenerationGraph:
)
validation = str(result)
# Simple validation check (in real implementation, parse the LLM response)
is_valid = "PASS" in validation.upper() or len(all_code) > 10
# Parse the LLM response: PASS only when PASS is present and FAIL is absent
is_valid = "PASS" in validation.upper() and "FAIL" not in validation.upper()
return {
"validation_result": {
+1 -1
View File
@@ -185,7 +185,7 @@ def run(
except UnsafeConfigurationError as exc:
typer.echo(f"Error: {exc}", err=True)
raise typer.Exit(code=1) from exc
except click.exceptions.Exit:
except (click.exceptions.Exit, typer.Exit):
raise
except CleverAgentsError as exc:
typer.echo(f"Error: {exc}", err=True)
+1 -1
View File
@@ -159,7 +159,7 @@ def run(
except UnsafeConfigurationError as exc:
typer.echo(f"Error: {exc}", err=True)
raise typer.Exit(code=1) from exc
except click.exceptions.Exit:
except (click.exceptions.Exit, typer.Exit):
raise
except CleverAgentsError as exc:
typer.echo(f"Error: {exc}", err=True)