feat(acms): implement pipeline Phase 3 components #622
@@ -0,0 +1,141 @@
|
||||
"""ASV benchmarks for ACMS pipeline Phase 3 — Context Finalization and
|
||||
advanced strategy components.
|
||||
|
||||
Measures the performance of:
|
||||
- RelevanceCoherenceOrderer with varying fragment counts
|
||||
- ProvenancePreambleGenerator with varying fragment counts
|
||||
- ArceStrategy with varying fragment counts and iteration limits
|
||||
- TemporalArchaeologyStrategy with varying fragment counts
|
||||
- PlanDecisionContextStrategy with varying fragment counts
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
import cleveragents # noqa: E402
|
||||
|
||||
importlib.reload(cleveragents)
|
||||
|
||||
from cleveragents.application.services.acms_advanced_strategies import ( # noqa: E402
|
||||
ArceStrategy,
|
||||
PlanDecisionContextStrategy,
|
||||
TemporalArchaeologyStrategy,
|
||||
)
|
||||
from cleveragents.application.services.acms_phase3 import ( # noqa: E402
|
||||
ProvenancePreambleGenerator,
|
||||
RelevanceCoherenceOrderer,
|
||||
)
|
||||
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
||||
ContextBudget,
|
||||
ContextFragment,
|
||||
FragmentProvenance,
|
||||
)
|
||||
|
||||
_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://phase3")
|
||||
|
||||
|
||||
def _make_fragments(
|
||||
n: int,
|
||||
*,
|
||||
unique_nodes: bool = True,
|
||||
vary_tiers: bool = False,
|
||||
) -> list[ContextFragment]:
|
||||
"""Create *n* fragments for benchmarks."""
|
||||
tiers = ["hot", "warm", "cold"]
|
||||
frags: list[ContextFragment] = []
|
||||
for i in range(n):
|
||||
node = f"bench://file{i}" if unique_nodes else "bench://same-node"
|
||||
tier = tiers[i % 3] if vary_tiers else "warm"
|
||||
frags.append(
|
||||
ContextFragment(
|
||||
uko_node=node,
|
||||
content=f"content-{i} " * max(1, (i % 10)),
|
||||
relevance_score=round(0.1 + (i % 10) * 0.09, 2),
|
||||
token_count=50 + (i % 5) * 10,
|
||||
detail_depth=i % 10,
|
||||
tier=tier,
|
||||
strategy_source=f"strategy-{i % 3}",
|
||||
provenance=_DEFAULT_PROV,
|
||||
)
|
||||
)
|
||||
return frags
|
||||
|
||||
|
||||
class RelevanceCoherenceOrdererSuite:
|
||||
"""Benchmark RelevanceCoherenceOrderer throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.orderer = RelevanceCoherenceOrderer()
|
||||
self.fragments = _make_fragments(n)
|
||||
|
||||
def time_order(self, n: int) -> None:
|
||||
self.orderer.order(self.fragments)
|
||||
|
||||
|
||||
class ProvenancePreambleGeneratorSuite:
|
||||
"""Benchmark ProvenancePreambleGenerator throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.generator = ProvenancePreambleGenerator()
|
||||
self.fragments = _make_fragments(n)
|
||||
|
||||
def time_generate(self, n: int) -> None:
|
||||
self.generator.generate(self.fragments)
|
||||
|
||||
|
||||
class ArceStrategySuite:
|
||||
"""Benchmark ArceStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = ArceStrategy(max_iterations=3)
|
||||
self.fragments = _make_fragments(n)
|
||||
self.budget = ContextBudget(max_tokens=100000, reserved_tokens=0)
|
||||
|
||||
def time_assemble(self, n: int) -> None:
|
||||
self.strategy.assemble(self.fragments, self.budget)
|
||||
|
||||
|
||||
class TemporalArchaeologyStrategySuite:
|
||||
"""Benchmark TemporalArchaeologyStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = TemporalArchaeologyStrategy()
|
||||
self.fragments = _make_fragments(n, vary_tiers=True)
|
||||
self.budget = ContextBudget(max_tokens=100000, reserved_tokens=0)
|
||||
|
||||
def time_assemble(self, n: int) -> None:
|
||||
self.strategy.assemble(self.fragments, self.budget)
|
||||
|
||||
|
||||
class PlanDecisionContextStrategySuite:
|
||||
"""Benchmark PlanDecisionContextStrategy throughput."""
|
||||
|
||||
params: list[int] = [10, 100, 500]
|
||||
param_names: list[str] = ["num_fragments"]
|
||||
|
||||
def setup(self, n: int) -> None:
|
||||
self.strategy = PlanDecisionContextStrategy()
|
||||
self.fragments = _make_fragments(n, vary_tiers=True)
|
||||
self.budget = ContextBudget(max_tokens=100000, reserved_tokens=0)
|
||||
|
||||
def time_assemble(self, n: int) -> None:
|
||||
self.strategy.assemble(self.fragments, self.budget)
|
||||
@@ -0,0 +1,320 @@
|
||||
@phase3 @acms @acms_pipeline_phase3
|
||||
Feature: ACMS Pipeline Phase 3 — Context Finalization and Advanced Strategies
|
||||
As a CleverAgents developer
|
||||
I want production-grade Phase 3 pipeline components and advanced strategies
|
||||
So that context assembly orders fragments coherently, generates provenance preambles,
|
||||
and supports ARCE, temporal archaeology, and plan decision context strategies
|
||||
|
||||
# ===========================================================================
|
||||
# RelevanceCoherenceOrderer
|
||||
# ===========================================================================
|
||||
|
||||
@orderer
|
||||
Scenario: Order fragments by relevance while grouping related nodes
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/io.py | io module | 0.9 | 20 | 3 |
|
||||
| project://app/main.py | main entry | 0.5 | 15 | 3 |
|
||||
| project://lib/util.py | utility fn | 0.7 | 25 | 3 |
|
||||
| project://app/net.py | network | 0.8 | 10 | 3 |
|
||||
When I order the fragments with RelevanceCoherenceOrderer
|
||||
Then the first ordered fragment should have uko_node "project://app/io.py"
|
||||
And fragments from the same group should be adjacent
|
||||
|
||||
@orderer
|
||||
Scenario: Order with empty fragment list
|
||||
Given an empty phase3 fragment list
|
||||
When I order the fragments with RelevanceCoherenceOrderer
|
||||
Then 0 fragments should remain after ordering
|
||||
|
||||
@orderer
|
||||
Scenario: Order single fragment returns it unchanged
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | hello | 0.5 | 10 | 3 |
|
||||
When I order the fragments with RelevanceCoherenceOrderer
|
||||
Then 1 fragment should remain after ordering
|
||||
|
||||
@orderer
|
||||
Scenario: Order preserves all fragments
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/a.py | alpha | 0.3 | 10 | 3 |
|
||||
| project://app/b.py | beta | 0.9 | 10 | 5 |
|
||||
| project://app/c.py | gamma | 0.6 | 10 | 1 |
|
||||
When I order the fragments with RelevanceCoherenceOrderer
|
||||
Then 3 fragments should remain after ordering
|
||||
|
||||
@orderer
|
||||
Scenario: Higher-depth fragments ordered after overview in same group
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | detail | 0.8 | 10 | 9 |
|
||||
| project://app/main.py | brief | 0.8 | 10 | 1 |
|
||||
When I order the fragments with RelevanceCoherenceOrderer
|
||||
Then 2 fragments should remain after ordering
|
||||
|
||||
# ===========================================================================
|
||||
# ProvenancePreambleGenerator
|
||||
# ===========================================================================
|
||||
|
||||
@preamble
|
||||
Scenario: Generate provenance preamble with strategy contributions
|
||||
Given the following phase3 fragments with strategy sources:
|
||||
| uko_node | content | score | tokens | depth | strategy_source |
|
||||
| project://app/main.py | hello | 0.8 | 100 | 3 | relevance |
|
||||
| project://app/io.py | world | 0.6 | 50 | 5 | relevance |
|
||||
| project://lib/util.py | util | 0.7 | 75 | 2 | arce |
|
||||
When I generate a preamble with ProvenancePreambleGenerator
|
||||
Then the preamble should contain "Strategy Contributions"
|
||||
And the preamble should contain "relevance"
|
||||
And the preamble should contain "arce"
|
||||
And the preamble should contain "Confidence"
|
||||
And the preamble should contain "Tier Distribution"
|
||||
And the preamble should contain "Depth Distribution"
|
||||
|
||||
@preamble
|
||||
Scenario: Generate preamble with confidence indicators
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | hello | 0.9 | 50 | 3 |
|
||||
| project://app/io.py | world | 0.3 | 50 | 5 |
|
||||
When I generate a preamble with ProvenancePreambleGenerator
|
||||
Then the preamble should contain "avg=0.600"
|
||||
And the preamble should contain "min=0.300"
|
||||
And the preamble should contain "max=0.900"
|
||||
|
||||
@preamble
|
||||
Scenario: Generate preamble returns None for empty fragments
|
||||
Given an empty phase3 fragment list
|
||||
When I generate a preamble with ProvenancePreambleGenerator
|
||||
Then the preamble should be None
|
||||
|
||||
@preamble
|
||||
Scenario: Preamble identifies coverage gaps for missing tiers
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | hello | 0.8 | 50 | 3 |
|
||||
When I generate a preamble with ProvenancePreambleGenerator
|
||||
Then the preamble should contain "Coverage Gaps"
|
||||
And the preamble should contain "Missing tiers"
|
||||
|
||||
@preamble
|
||||
Scenario: Preamble includes UKO node count
|
||||
Given the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/a.py | alpha | 0.8 | 50 | 3 |
|
||||
| project://app/b.py | beta | 0.7 | 50 | 5 |
|
||||
| project://lib/c.py | gamma | 0.6 | 50 | 2 |
|
||||
When I generate a preamble with ProvenancePreambleGenerator
|
||||
Then the preamble should contain "UKO Nodes: 3"
|
||||
|
||||
# ===========================================================================
|
||||
# ArceStrategy
|
||||
# ===========================================================================
|
||||
|
||||
@arce
|
||||
Scenario: ARCE ranks fragments with iterative refinement
|
||||
Given an ArceStrategy with max_iterations 3
|
||||
And the following strategy3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/io.py | async IO handler | 0.7 | 20 | 5 |
|
||||
| project://app/main.py | main application entry | 0.5 | 15 | 3 |
|
||||
| project://lib/util.py | utility library | 0.3 | 25 | 2 |
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the ArceStrategy
|
||||
Then fragments should be returned by arce strategy
|
||||
And the arce result should respect the budget
|
||||
|
||||
@arce
|
||||
Scenario: ARCE returns empty for empty input
|
||||
Given an ArceStrategy with max_iterations 5
|
||||
And an empty strategy3 fragment list
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the ArceStrategy
|
||||
Then 0 fragments should be returned by arce strategy
|
||||
|
||||
@arce
|
||||
Scenario: ARCE respects budget constraint
|
||||
Given an ArceStrategy with max_iterations 3
|
||||
And the following strategy3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/a.py | alpha | 0.9 | 100 | 5 |
|
||||
| project://app/b.py | beta | 0.7 | 100 | 3 |
|
||||
| project://app/c.py | gamma | 0.5 | 100 | 2 |
|
||||
And a strategy3 budget with max_tokens 250 and reserved_tokens 0
|
||||
When I assemble with the ArceStrategy
|
||||
Then at most 2 fragments should be returned by arce strategy
|
||||
|
||||
@arce
|
||||
Scenario: ARCE can_handle returns 0.95
|
||||
Given an ArceStrategy with max_iterations 5
|
||||
When I check can_handle on ArceStrategy
|
||||
Then the arce confidence should be 0.95
|
||||
|
||||
@arce
|
||||
Scenario: ARCE reports capabilities
|
||||
Given an ArceStrategy with max_iterations 5
|
||||
Then the ArceStrategy name should be "arce"
|
||||
And the ArceStrategy should support semantic search
|
||||
|
||||
@arce
|
||||
Scenario: ARCE explain returns description
|
||||
Given an ArceStrategy with max_iterations 5
|
||||
Then the ArceStrategy explain should contain "Adaptive Recursive"
|
||||
|
||||
@arce
|
||||
Scenario: ARCE iteration limit prevents unbounded refinement
|
||||
Given an ArceStrategy with max_iterations 2
|
||||
And the following strategy3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/a.py | alpha content | 0.9 | 50 | 5 |
|
||||
| project://app/b.py | beta content | 0.7 | 50 | 3 |
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the ArceStrategy
|
||||
Then fragments should be returned by arce strategy
|
||||
|
||||
# ===========================================================================
|
||||
# TemporalArchaeologyStrategy
|
||||
# ===========================================================================
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology prioritises cold-tier fragments
|
||||
Given a TemporalArchaeologyStrategy
|
||||
And the following strategy3 fragments with tiers:
|
||||
| uko_node | content | score | tokens | depth | tier |
|
||||
| project://app/old.py | archived | 0.5 | 20 | 3 | cold |
|
||||
| project://app/new.py | recent | 0.9 | 15 | 3 | hot |
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the TemporalArchaeologyStrategy
|
||||
Then the first temporal result fragment should have uko_node "project://app/old.py"
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology returns empty for empty input
|
||||
Given a TemporalArchaeologyStrategy
|
||||
And an empty strategy3 fragment list
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the TemporalArchaeologyStrategy
|
||||
Then 0 fragments should be returned by temporal strategy
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology can_handle returns 0.5
|
||||
Given a TemporalArchaeologyStrategy
|
||||
When I check can_handle on TemporalArchaeologyStrategy
|
||||
Then the temporal confidence should be 0.5
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology reports capabilities
|
||||
Given a TemporalArchaeologyStrategy
|
||||
Then the TemporalArchaeologyStrategy name should be "temporal-archaeology"
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology explain returns description
|
||||
Given a TemporalArchaeologyStrategy
|
||||
Then the TemporalArchaeologyStrategy explain should contain "Historical"
|
||||
|
||||
@temporal
|
||||
Scenario: TemporalArchaeology respects budget
|
||||
Given a TemporalArchaeologyStrategy
|
||||
And the following strategy3 fragments with tiers:
|
||||
| uko_node | content | score | tokens | depth | tier |
|
||||
| project://app/a.py | alpha | 0.9 | 100 | 3 | cold |
|
||||
| project://app/b.py | beta | 0.7 | 100 | 3 | cold |
|
||||
| project://app/c.py | gamma | 0.5 | 100 | 3 | warm |
|
||||
And a strategy3 budget with max_tokens 250 and reserved_tokens 0
|
||||
When I assemble with the TemporalArchaeologyStrategy
|
||||
Then at most 2 fragments should be returned by temporal strategy
|
||||
|
||||
# ===========================================================================
|
||||
# PlanDecisionContextStrategy
|
||||
# ===========================================================================
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext prioritises warm-tier fragments
|
||||
Given a PlanDecisionContextStrategy
|
||||
And the following strategy3 fragments with tiers:
|
||||
| uko_node | content | score | tokens | depth | tier |
|
||||
| project://app/plan.py | decision | 0.7 | 20 | 3 | warm |
|
||||
| project://app/new.py | new code | 0.9 | 15 | 3 | hot |
|
||||
| project://app/old.py | archived | 0.5 | 25 | 3 | cold |
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the PlanDecisionContextStrategy
|
||||
Then the first plan_decision result should have uko_node "project://app/plan.py"
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext returns empty for empty input
|
||||
Given a PlanDecisionContextStrategy
|
||||
And an empty strategy3 fragment list
|
||||
And a strategy3 budget with max_tokens 1000 and reserved_tokens 0
|
||||
When I assemble with the PlanDecisionContextStrategy
|
||||
Then 0 fragments should be returned by plan_decision strategy
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext can_handle returns 0.7
|
||||
Given a PlanDecisionContextStrategy
|
||||
When I check can_handle on PlanDecisionContextStrategy
|
||||
Then the plan_decision confidence should be 0.7
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext reports capabilities
|
||||
Given a PlanDecisionContextStrategy
|
||||
Then the PlanDecisionContextStrategy name should be "plan-decision-context"
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext explain returns description
|
||||
Given a PlanDecisionContextStrategy
|
||||
Then the PlanDecisionContextStrategy explain should contain "decision"
|
||||
|
||||
@plan_decision
|
||||
Scenario: PlanDecisionContext respects budget
|
||||
Given a PlanDecisionContextStrategy
|
||||
And the following strategy3 fragments with tiers:
|
||||
| uko_node | content | score | tokens | depth | tier |
|
||||
| project://app/a.py | alpha | 0.9 | 100 | 3 | warm |
|
||||
| project://app/b.py | beta | 0.7 | 100 | 3 | cold |
|
||||
| project://app/c.py | gamma | 0.5 | 100 | 3 | hot |
|
||||
And a strategy3 budget with max_tokens 250 and reserved_tokens 0
|
||||
When I assemble with the PlanDecisionContextStrategy
|
||||
Then at most 2 fragments should be returned by plan_decision strategy
|
||||
|
||||
# ===========================================================================
|
||||
# Pipeline Registration
|
||||
# ===========================================================================
|
||||
|
||||
@registration
|
||||
Scenario: Register ArceStrategy with pipeline
|
||||
Given an ACMS pipeline for phase3 strategy tests
|
||||
When I register ArceStrategy with the pipeline
|
||||
Then the phase3 pipeline should have strategy "arce"
|
||||
|
||||
@registration
|
||||
Scenario: Register all Phase 3 strategies with pipeline
|
||||
Given an ACMS pipeline for phase3 strategy tests
|
||||
When I register all phase3 strategies with the pipeline
|
||||
Then the phase3 pipeline should have strategy "arce"
|
||||
And the phase3 pipeline should have strategy "temporal-archaeology"
|
||||
And the phase3 pipeline should have strategy "plan-decision-context"
|
||||
|
||||
# ===========================================================================
|
||||
# Pipeline Integration — DI injection of Phase 3 components
|
||||
# ===========================================================================
|
||||
|
||||
@pipeline_integration
|
||||
Scenario: Inject RelevanceCoherenceOrderer into pipeline
|
||||
Given the ACMS pipeline with a RelevanceCoherenceOrderer
|
||||
And the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | hello | 0.8 | 10 | 3 |
|
||||
| project://lib/util.py | world | 0.6 | 15 | 5 |
|
||||
When I assemble context through the phase3 pipeline
|
||||
Then the phase3 pipeline output should contain 2 fragments
|
||||
|
||||
@pipeline_integration
|
||||
Scenario: Inject ProvenancePreambleGenerator into pipeline
|
||||
Given the ACMS pipeline with a ProvenancePreambleGenerator
|
||||
And the following phase3 fragments:
|
||||
| uko_node | content | score | tokens | depth |
|
||||
| project://app/main.py | hello | 0.8 | 10 | 3 |
|
||||
When I assemble context through the phase3 pipeline
|
||||
Then the phase3 pipeline output should have a preamble
|
||||
And the phase3 pipeline preamble should contain "Context Assembly Provenance"
|
||||
@@ -0,0 +1,503 @@
|
||||
"""Step definitions for features/acms_pipeline_phase3.feature.
|
||||
|
||||
Tests the ACMS pipeline Phase 3 (Context Finalization) components and
|
||||
advanced context strategies directly in-memory — no database required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from behave import given, then, when
|
||||
from behave.runner import Context
|
||||
|
||||
from cleveragents.application.services.acms_advanced_strategies import (
|
||||
ArceStrategy,
|
||||
PlanDecisionContextStrategy,
|
||||
TemporalArchaeologyStrategy,
|
||||
)
|
||||
from cleveragents.application.services.acms_phase3 import (
|
||||
ProvenancePreambleGenerator,
|
||||
RelevanceCoherenceOrderer,
|
||||
)
|
||||
from cleveragents.application.services.acms_service import ACMSPipeline
|
||||
from cleveragents.domain.models.core.context_fragment import (
|
||||
ContextBudget,
|
||||
ContextFragment,
|
||||
FragmentProvenance,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_PROVENANCE = FragmentProvenance(resource_uri="test://phase3")
|
||||
_TEST_PLAN_ID = "01JQTESTPN00000000000000DD"
|
||||
|
||||
|
||||
def _make_phase3_fragment(**kwargs: Any) -> ContextFragment:
|
||||
"""Create a ContextFragment with sensible Phase 3 test defaults."""
|
||||
kwargs.setdefault("uko_node", "test://default")
|
||||
kwargs.setdefault("token_count", 0)
|
||||
kwargs.setdefault("provenance", _DEFAULT_PROVENANCE)
|
||||
return ContextFragment(**kwargs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fragment list construction — Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the following phase3 fragments:")
|
||||
def step_given_phase3_fragments(context: Context) -> None:
|
||||
frags: list[ContextFragment] = []
|
||||
for row in context.table:
|
||||
frags.append(
|
||||
_make_phase3_fragment(
|
||||
uko_node=row["uko_node"],
|
||||
content=row["content"],
|
||||
relevance_score=float(row["score"]),
|
||||
token_count=int(row["tokens"]),
|
||||
detail_depth=int(row["depth"]),
|
||||
)
|
||||
)
|
||||
context.phase3_fragments = frags
|
||||
|
||||
|
||||
@given("the following phase3 fragments with strategy sources:")
|
||||
def step_given_phase3_fragments_with_strategy(context: Context) -> None:
|
||||
frags: list[ContextFragment] = []
|
||||
for row in context.table:
|
||||
frags.append(
|
||||
_make_phase3_fragment(
|
||||
uko_node=row["uko_node"],
|
||||
content=row["content"],
|
||||
relevance_score=float(row["score"]),
|
||||
token_count=int(row["tokens"]),
|
||||
detail_depth=int(row["depth"]),
|
||||
strategy_source=row["strategy_source"],
|
||||
)
|
||||
)
|
||||
context.phase3_fragments = frags
|
||||
|
||||
|
||||
@given("an empty phase3 fragment list")
|
||||
def step_given_empty_phase3(context: Context) -> None:
|
||||
context.phase3_fragments = []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategy fragment construction — Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the following strategy3 fragments:")
|
||||
def step_given_strategy3_fragments(context: Context) -> None:
|
||||
frags: list[ContextFragment] = []
|
||||
for row in context.table:
|
||||
frags.append(
|
||||
_make_phase3_fragment(
|
||||
uko_node=row["uko_node"],
|
||||
content=row["content"],
|
||||
relevance_score=float(row["score"]),
|
||||
token_count=int(row["tokens"]),
|
||||
detail_depth=int(row["depth"]),
|
||||
)
|
||||
)
|
||||
context.strategy3_fragments = frags
|
||||
|
||||
|
||||
@given("the following strategy3 fragments with tiers:")
|
||||
def step_given_strategy3_fragments_with_tiers(context: Context) -> None:
|
||||
frags: list[ContextFragment] = []
|
||||
for row in context.table:
|
||||
frags.append(
|
||||
_make_phase3_fragment(
|
||||
uko_node=row["uko_node"],
|
||||
content=row["content"],
|
||||
relevance_score=float(row["score"]),
|
||||
token_count=int(row["tokens"]),
|
||||
detail_depth=int(row["depth"]),
|
||||
tier=row["tier"],
|
||||
)
|
||||
)
|
||||
context.strategy3_fragments = frags
|
||||
|
||||
|
||||
@given("an empty strategy3 fragment list")
|
||||
def step_given_empty_strategy3(context: Context) -> None:
|
||||
context.strategy3_fragments = []
|
||||
|
||||
|
||||
@given("a strategy3 budget with max_tokens {max_t:d} and reserved_tokens {res_t:d}")
|
||||
def step_given_strategy3_budget(context: Context, max_t: int, res_t: int) -> None:
|
||||
context.strategy3_budget = ContextBudget(max_tokens=max_t, reserved_tokens=res_t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Strategy constructors — Given steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("an ArceStrategy with max_iterations {max_iter:d}")
|
||||
def step_given_arce_strategy(context: Context, max_iter: int) -> None:
|
||||
context.arce_strategy = ArceStrategy(max_iterations=max_iter)
|
||||
|
||||
|
||||
@given("a TemporalArchaeologyStrategy")
|
||||
def step_given_temporal_strategy(context: Context) -> None:
|
||||
context.temporal_strategy = TemporalArchaeologyStrategy()
|
||||
|
||||
|
||||
@given("a PlanDecisionContextStrategy")
|
||||
def step_given_plan_decision_strategy(context: Context) -> None:
|
||||
context.plan_decision_strategy = PlanDecisionContextStrategy()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RelevanceCoherenceOrderer — When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I order the fragments with RelevanceCoherenceOrderer")
|
||||
def step_order_fragments(context: Context) -> None:
|
||||
orderer = RelevanceCoherenceOrderer()
|
||||
context.phase3_result = list(orderer.order(context.phase3_fragments))
|
||||
|
||||
|
||||
@then('the first ordered fragment should have uko_node "{expected}"')
|
||||
def step_first_ordered_node(context: Context, expected: str) -> None:
|
||||
assert len(context.phase3_result) >= 1
|
||||
assert context.phase3_result[0].uko_node == expected, (
|
||||
f"Expected {expected}, got {context.phase3_result[0].uko_node}"
|
||||
)
|
||||
|
||||
|
||||
@then("fragments from the same group should be adjacent")
|
||||
def step_same_group_adjacent(context: Context) -> None:
|
||||
# Verify that fragments with the same prefix are adjacent
|
||||
seen_prefixes: set[str] = set()
|
||||
current_prefix: str | None = None
|
||||
for frag in context.phase3_result:
|
||||
prefix = _extract_prefix(frag.uko_node)
|
||||
if prefix != current_prefix:
|
||||
assert prefix not in seen_prefixes, (
|
||||
f"Prefix {prefix} appeared non-contiguously"
|
||||
)
|
||||
if current_prefix is not None:
|
||||
seen_prefixes.add(current_prefix)
|
||||
current_prefix = prefix
|
||||
|
||||
|
||||
@then("{count:d} fragments should remain after ordering")
|
||||
def step_order_count_plural(context: Context, count: int) -> None:
|
||||
assert len(context.phase3_result) == count, (
|
||||
f"Expected {count}, got {len(context.phase3_result)}"
|
||||
)
|
||||
|
||||
|
||||
@then("{count:d} fragment should remain after ordering")
|
||||
def step_order_count_singular(context: Context, count: int) -> None:
|
||||
assert len(context.phase3_result) == count, (
|
||||
f"Expected {count}, got {len(context.phase3_result)}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ProvenancePreambleGenerator — When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I generate a preamble with ProvenancePreambleGenerator")
|
||||
def step_generate_preamble(context: Context) -> None:
|
||||
generator = ProvenancePreambleGenerator()
|
||||
context.phase3_preamble = generator.generate(context.phase3_fragments)
|
||||
|
||||
|
||||
@then('the preamble should contain "{text}"')
|
||||
def step_preamble_contains(context: Context, text: str) -> None:
|
||||
assert context.phase3_preamble is not None, "Preamble was None"
|
||||
assert text in context.phase3_preamble, (
|
||||
f"Expected preamble to contain '{text}', got:\n{context.phase3_preamble}"
|
||||
)
|
||||
|
||||
|
||||
@then("the preamble should be None")
|
||||
def step_preamble_is_none(context: Context) -> None:
|
||||
assert context.phase3_preamble is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ArceStrategy — When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I assemble with the ArceStrategy")
|
||||
def step_assemble_arce(context: Context) -> None:
|
||||
context.arce_result = list(
|
||||
context.arce_strategy.assemble(
|
||||
context.strategy3_fragments,
|
||||
context.strategy3_budget,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@when("I check can_handle on ArceStrategy")
|
||||
def step_check_arce_can_handle(context: Context) -> None:
|
||||
context.arce_confidence = context.arce_strategy.can_handle({})
|
||||
|
||||
|
||||
@then("fragments should be returned by arce strategy")
|
||||
def step_arce_has_results(context: Context) -> None:
|
||||
assert len(context.arce_result) > 0
|
||||
|
||||
|
||||
@then("the arce result should respect the budget")
|
||||
def step_arce_respects_budget(context: Context) -> None:
|
||||
total = sum(f.token_count for f in context.arce_result)
|
||||
available = context.strategy3_budget.available_tokens
|
||||
assert total <= available, f"ARCE used {total} tokens, budget was {available}"
|
||||
|
||||
|
||||
@then("{count:d} fragments should be returned by arce strategy")
|
||||
def step_arce_exact_count(context: Context, count: int) -> None:
|
||||
assert len(context.arce_result) == count
|
||||
|
||||
|
||||
@then("at most {count:d} fragments should be returned by arce strategy")
|
||||
def step_arce_at_most_count(context: Context, count: int) -> None:
|
||||
assert len(context.arce_result) <= count, (
|
||||
f"Expected at most {count}, got {len(context.arce_result)}"
|
||||
)
|
||||
|
||||
|
||||
@then("the arce confidence should be {expected:g}")
|
||||
def step_arce_confidence(context: Context, expected: float) -> None:
|
||||
assert context.arce_confidence == expected
|
||||
|
||||
|
||||
@then('the ArceStrategy name should be "{expected}"')
|
||||
def step_arce_name(context: Context, expected: str) -> None:
|
||||
assert context.arce_strategy.name == expected
|
||||
|
||||
|
||||
@then("the ArceStrategy should support semantic search")
|
||||
def step_arce_supports_semantic(context: Context) -> None:
|
||||
assert context.arce_strategy.capabilities.supports_semantic_search
|
||||
|
||||
|
||||
@then('the ArceStrategy explain should contain "{text}"')
|
||||
def step_arce_explain(context: Context, text: str) -> None:
|
||||
explanation = context.arce_strategy.explain()
|
||||
assert text in explanation, (
|
||||
f"Expected explain to contain '{text}', got: {explanation}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TemporalArchaeologyStrategy — When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I assemble with the TemporalArchaeologyStrategy")
|
||||
def step_assemble_temporal(context: Context) -> None:
|
||||
context.temporal_result = list(
|
||||
context.temporal_strategy.assemble(
|
||||
context.strategy3_fragments,
|
||||
context.strategy3_budget,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@when("I check can_handle on TemporalArchaeologyStrategy")
|
||||
def step_check_temporal_can_handle(context: Context) -> None:
|
||||
context.temporal_confidence = context.temporal_strategy.can_handle({})
|
||||
|
||||
|
||||
@then('the first temporal result fragment should have uko_node "{expected}"')
|
||||
def step_temporal_first_node(context: Context, expected: str) -> None:
|
||||
assert len(context.temporal_result) >= 1
|
||||
assert context.temporal_result[0].uko_node == expected, (
|
||||
f"Expected {expected}, got {context.temporal_result[0].uko_node}"
|
||||
)
|
||||
|
||||
|
||||
@then("{count:d} fragments should be returned by temporal strategy")
|
||||
def step_temporal_exact_count(context: Context, count: int) -> None:
|
||||
assert len(context.temporal_result) == count
|
||||
|
||||
|
||||
@then("at most {count:d} fragments should be returned by temporal strategy")
|
||||
def step_temporal_at_most_count(context: Context, count: int) -> None:
|
||||
assert len(context.temporal_result) <= count
|
||||
|
||||
|
||||
@then("the temporal confidence should be {expected:g}")
|
||||
def step_temporal_confidence(context: Context, expected: float) -> None:
|
||||
assert context.temporal_confidence == expected
|
||||
|
||||
|
||||
@then('the TemporalArchaeologyStrategy name should be "{expected}"')
|
||||
def step_temporal_name(context: Context, expected: str) -> None:
|
||||
assert context.temporal_strategy.name == expected
|
||||
|
||||
|
||||
@then('the TemporalArchaeologyStrategy explain should contain "{text}"')
|
||||
def step_temporal_explain(context: Context, text: str) -> None:
|
||||
explanation = context.temporal_strategy.explain()
|
||||
assert text in explanation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PlanDecisionContextStrategy — When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@when("I assemble with the PlanDecisionContextStrategy")
|
||||
def step_assemble_plan_decision(context: Context) -> None:
|
||||
context.plan_decision_result = list(
|
||||
context.plan_decision_strategy.assemble(
|
||||
context.strategy3_fragments,
|
||||
context.strategy3_budget,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@when("I check can_handle on PlanDecisionContextStrategy")
|
||||
def step_check_plan_decision_can_handle(context: Context) -> None:
|
||||
context.plan_decision_confidence = context.plan_decision_strategy.can_handle({})
|
||||
|
||||
|
||||
@then('the first plan_decision result should have uko_node "{expected}"')
|
||||
def step_plan_decision_first_node(context: Context, expected: str) -> None:
|
||||
assert len(context.plan_decision_result) >= 1
|
||||
assert context.plan_decision_result[0].uko_node == expected, (
|
||||
f"Expected {expected}, got {context.plan_decision_result[0].uko_node}"
|
||||
)
|
||||
|
||||
|
||||
@then("{count:d} fragments should be returned by plan_decision strategy")
|
||||
def step_plan_decision_exact_count(context: Context, count: int) -> None:
|
||||
assert len(context.plan_decision_result) == count
|
||||
|
||||
|
||||
@then("at most {count:d} fragments should be returned by plan_decision strategy")
|
||||
def step_plan_decision_at_most_count(context: Context, count: int) -> None:
|
||||
assert len(context.plan_decision_result) <= count
|
||||
|
||||
|
||||
@then("the plan_decision confidence should be {expected:g}")
|
||||
def step_plan_decision_confidence(context: Context, expected: float) -> None:
|
||||
assert context.plan_decision_confidence == expected
|
||||
|
||||
|
||||
@then('the PlanDecisionContextStrategy name should be "{expected}"')
|
||||
def step_plan_decision_name(context: Context, expected: str) -> None:
|
||||
assert context.plan_decision_strategy.name == expected
|
||||
|
||||
|
||||
@then('the PlanDecisionContextStrategy explain should contain "{text}"')
|
||||
def step_plan_decision_explain(context: Context, text: str) -> None:
|
||||
explanation = context.plan_decision_strategy.explain()
|
||||
assert text in explanation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipeline Registration — Given / When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("an ACMS pipeline for phase3 strategy tests")
|
||||
def step_given_phase3_pipeline(context: Context) -> None:
|
||||
context.phase3_test_pipeline = ACMSPipeline()
|
||||
|
||||
|
||||
@when("I register ArceStrategy with the pipeline")
|
||||
def step_register_arce(context: Context) -> None:
|
||||
strategy = ArceStrategy()
|
||||
context.phase3_test_pipeline.register_strategy("arce", strategy)
|
||||
|
||||
|
||||
@when("I register all phase3 strategies with the pipeline")
|
||||
def step_register_all_phase3(context: Context) -> None:
|
||||
context.phase3_test_pipeline.register_strategy("arce", ArceStrategy())
|
||||
context.phase3_test_pipeline.register_strategy(
|
||||
"temporal-archaeology", TemporalArchaeologyStrategy()
|
||||
)
|
||||
context.phase3_test_pipeline.register_strategy(
|
||||
"plan-decision-context", PlanDecisionContextStrategy()
|
||||
)
|
||||
|
||||
|
||||
@then('the phase3 pipeline should have strategy "{name}"')
|
||||
def step_phase3_pipeline_has_strategy(context: Context, name: str) -> None:
|
||||
assert name in context.phase3_test_pipeline._strategies
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipeline Integration — Given / When / Then
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@given("the ACMS pipeline with a RelevanceCoherenceOrderer")
|
||||
def step_pipeline_with_orderer(context: Context) -> None:
|
||||
context.phase3_pipeline = ACMSPipeline(
|
||||
orderer=RelevanceCoherenceOrderer(),
|
||||
)
|
||||
|
||||
|
||||
@given("the ACMS pipeline with a ProvenancePreambleGenerator")
|
||||
def step_pipeline_with_preamble(context: Context) -> None:
|
||||
context.phase3_pipeline = ACMSPipeline(
|
||||
preamble_generator=ProvenancePreambleGenerator(),
|
||||
)
|
||||
|
||||
|
||||
@when("I assemble context through the phase3 pipeline")
|
||||
def step_assemble_phase3_pipeline(context: Context) -> None:
|
||||
budget = ContextBudget(max_tokens=100000, reserved_tokens=0)
|
||||
payload = context.phase3_pipeline.assemble(
|
||||
plan_id=_TEST_PLAN_ID,
|
||||
fragments=context.phase3_fragments,
|
||||
budget=budget,
|
||||
)
|
||||
context.phase3_payload = payload
|
||||
|
||||
|
||||
@then("the phase3 pipeline output should contain {count:d} fragments")
|
||||
def step_phase3_pipeline_count_plural(context: Context, count: int) -> None:
|
||||
assert len(context.phase3_payload.fragments) == count, (
|
||||
f"Expected {count}, got {len(context.phase3_payload.fragments)}"
|
||||
)
|
||||
|
||||
|
||||
@then("the phase3 pipeline output should contain {count:d} fragment")
|
||||
def step_phase3_pipeline_count_singular(context: Context, count: int) -> None:
|
||||
assert len(context.phase3_payload.fragments) == count
|
||||
|
||||
|
||||
@then("the phase3 pipeline output should have a preamble")
|
||||
def step_phase3_pipeline_has_preamble(context: Context) -> None:
|
||||
assert context.phase3_payload.preamble is not None, (
|
||||
"Expected pipeline output to have a preamble"
|
||||
)
|
||||
|
||||
|
||||
@then('the phase3 pipeline preamble should contain "{text}"')
|
||||
def step_phase3_pipeline_preamble_contains(context: Context, text: str) -> None:
|
||||
preamble = context.phase3_payload.preamble
|
||||
assert preamble is not None, "Preamble was None"
|
||||
assert text in preamble, f"Expected preamble to contain '{text}', got:\n{preamble}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_prefix(uko_node: str) -> str:
|
||||
"""Extract grouping prefix for adjacency checks."""
|
||||
if "://" in uko_node:
|
||||
uko_node = uko_node.split("://", 1)[1]
|
||||
segments = [s for s in uko_node.replace("\\", "/").split("/") if s]
|
||||
return "/".join(segments[:2]) if len(segments) >= 2 else uko_node
|
||||
@@ -0,0 +1,57 @@
|
||||
*** Settings ***
|
||||
Documentation Integration smoke tests for ACMS pipeline Phase 3 components
|
||||
Resource ${CURDIR}/common.resource
|
||||
Suite Setup Setup Test Environment
|
||||
Suite Teardown Cleanup Test Environment
|
||||
|
||||
*** Variables ***
|
||||
${HELPER} ${CURDIR}/helper_acms_pipeline_phase3.py
|
||||
|
||||
*** Test Cases ***
|
||||
RelevanceCoherenceOrderer Orders Fragments
|
||||
[Documentation] Order fragments by relevance while maintaining narrative coherence
|
||||
${result}= Run Process ${PYTHON} ${HELPER} orderer-basic cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-orderer-ok
|
||||
|
||||
ProvenancePreambleGenerator Generates Preamble
|
||||
[Documentation] Generate structured provenance preamble from fragments
|
||||
${result}= Run Process ${PYTHON} ${HELPER} preamble-generate cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-preamble-ok
|
||||
|
||||
ArceStrategy Iterative Refinement
|
||||
[Documentation] ARCE strategy with multi-backend iterative refinement
|
||||
${result}= Run Process ${PYTHON} ${HELPER} arce-strategy cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-arce-ok
|
||||
|
||||
TemporalArchaeologyStrategy Cold Tier Preference
|
||||
[Documentation] Temporal archaeology strategy prioritises cold-tier fragments
|
||||
${result}= Run Process ${PYTHON} ${HELPER} temporal-strategy cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-temporal-ok
|
||||
|
||||
PlanDecisionContextStrategy Warm Tier Preference
|
||||
[Documentation] Plan decision context strategy prioritises warm/cold tiers
|
||||
${result}= Run Process ${PYTHON} ${HELPER} plan-decision-strategy cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-plan-decision-ok
|
||||
|
||||
Pipeline Integration With Phase3 Components
|
||||
[Documentation] Inject Phase 3 components into ACMSPipeline
|
||||
${result}= Run Process ${PYTHON} ${HELPER} pipeline-integration cwd=${WORKSPACE}
|
||||
Log ${result.stdout}
|
||||
Log ${result.stderr}
|
||||
Should Be Equal As Integers ${result.rc} 0
|
||||
Should Contain ${result.stdout} phase3-pipeline-ok
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Robot Framework helper for ACMS pipeline Phase 3 integration tests.
|
||||
|
||||
Provides a CLI-style interface for Robot to invoke Phase 3 pipeline
|
||||
components and verify the results. Exit code 0 = success, 1 = failure.
|
||||
|
||||
Usage:
|
||||
python robot/helper_acms_pipeline_phase3.py orderer-basic
|
||||
python robot/helper_acms_pipeline_phase3.py preamble-generate
|
||||
python robot/helper_acms_pipeline_phase3.py arce-strategy
|
||||
python robot/helper_acms_pipeline_phase3.py temporal-strategy
|
||||
python robot/helper_acms_pipeline_phase3.py plan-decision-strategy
|
||||
python robot/helper_acms_pipeline_phase3.py pipeline-integration
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
_SRC = str(Path(__file__).resolve().parents[1] / "src")
|
||||
if _SRC not in sys.path:
|
||||
sys.path.insert(0, _SRC)
|
||||
|
||||
from cleveragents.application.services.acms_advanced_strategies import ( # noqa: E402
|
||||
ArceStrategy,
|
||||
PlanDecisionContextStrategy,
|
||||
TemporalArchaeologyStrategy,
|
||||
)
|
||||
from cleveragents.application.services.acms_phase3 import ( # noqa: E402
|
||||
ProvenancePreambleGenerator,
|
||||
RelevanceCoherenceOrderer,
|
||||
)
|
||||
from cleveragents.application.services.acms_service import ACMSPipeline # noqa: E402
|
||||
from cleveragents.domain.models.core.context_fragment import ( # noqa: E402
|
||||
ContextBudget,
|
||||
ContextFragment,
|
||||
FragmentProvenance,
|
||||
)
|
||||
|
||||
_DEFAULT_PROV = FragmentProvenance(resource_uri="test://robot-phase3")
|
||||
_PLAN_ID = "01JQTESTPN00000000000000DD"
|
||||
|
||||
|
||||
def _frag(
|
||||
uko_node: str = "test://default",
|
||||
content: str = "test content",
|
||||
score: float = 0.5,
|
||||
tokens: int = 10,
|
||||
depth: int = 0,
|
||||
tier: str = "warm",
|
||||
strategy_source: str = "",
|
||||
) -> ContextFragment:
|
||||
return ContextFragment(
|
||||
uko_node=uko_node,
|
||||
content=content,
|
||||
relevance_score=score,
|
||||
token_count=tokens,
|
||||
detail_depth=depth,
|
||||
tier=tier,
|
||||
strategy_source=strategy_source,
|
||||
provenance=_DEFAULT_PROV,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _test_orderer_basic() -> None:
|
||||
"""Test RelevanceCoherenceOrderer groups and orders fragments."""
|
||||
orderer = RelevanceCoherenceOrderer()
|
||||
frags = [
|
||||
_frag("project://app/io.py", "io module", 0.9, 20, 3),
|
||||
_frag("project://app/main.py", "main entry", 0.5, 15, 3),
|
||||
_frag("project://lib/util.py", "utility", 0.7, 25, 3),
|
||||
_frag("project://app/net.py", "network", 0.8, 10, 3),
|
||||
]
|
||||
result = list(orderer.order(frags))
|
||||
assert len(result) == 4, f"Expected 4, got {len(result)}"
|
||||
# First fragment should be from the highest-relevance group
|
||||
assert result[0].relevance_score == 0.9
|
||||
print("phase3-orderer-ok")
|
||||
|
||||
|
||||
def _test_preamble_generate() -> None:
|
||||
"""Test ProvenancePreambleGenerator produces structured preamble."""
|
||||
gen = ProvenancePreambleGenerator()
|
||||
frags = [
|
||||
_frag(
|
||||
"project://app/main.py",
|
||||
"hello",
|
||||
0.8,
|
||||
100,
|
||||
3,
|
||||
strategy_source="relevance",
|
||||
),
|
||||
_frag(
|
||||
"project://app/io.py",
|
||||
"world",
|
||||
0.6,
|
||||
50,
|
||||
5,
|
||||
strategy_source="arce",
|
||||
),
|
||||
]
|
||||
preamble = gen.generate(frags)
|
||||
assert preamble is not None
|
||||
assert "Strategy Contributions" in preamble
|
||||
assert "relevance" in preamble
|
||||
assert "arce" in preamble
|
||||
assert "Confidence" in preamble
|
||||
|
||||
# Empty list returns None
|
||||
assert gen.generate([]) is None
|
||||
print("phase3-preamble-ok")
|
||||
|
||||
|
||||
def _test_arce_strategy() -> None:
|
||||
"""Test ArceStrategy iterative refinement."""
|
||||
strategy = ArceStrategy(max_iterations=3)
|
||||
assert strategy.name == "arce"
|
||||
assert strategy.can_handle({}) == 0.95
|
||||
assert strategy.capabilities.supports_semantic_search
|
||||
|
||||
frags = [
|
||||
_frag("project://app/io.py", "async IO handler", 0.7, 20, 5),
|
||||
_frag("project://app/main.py", "main application", 0.5, 15, 3),
|
||||
]
|
||||
budget = ContextBudget(max_tokens=1000, reserved_tokens=0)
|
||||
result = list(strategy.assemble(frags, budget))
|
||||
assert len(result) > 0
|
||||
total = sum(f.token_count for f in result)
|
||||
assert total <= 1000
|
||||
|
||||
# Empty input
|
||||
assert list(strategy.assemble([], budget)) == []
|
||||
print("phase3-arce-ok")
|
||||
|
||||
|
||||
def _test_temporal_strategy() -> None:
|
||||
"""Test TemporalArchaeologyStrategy cold-tier preference."""
|
||||
strategy = TemporalArchaeologyStrategy()
|
||||
assert strategy.name == "temporal-archaeology"
|
||||
assert strategy.can_handle({}) == 0.5
|
||||
|
||||
frags = [
|
||||
_frag("project://app/old.py", "archived", 0.5, 20, 3, tier="cold"),
|
||||
_frag("project://app/new.py", "recent", 0.9, 15, 3, tier="hot"),
|
||||
]
|
||||
budget = ContextBudget(max_tokens=1000, reserved_tokens=0)
|
||||
result = list(strategy.assemble(frags, budget))
|
||||
assert len(result) > 0
|
||||
# Cold-tier should be first
|
||||
assert result[0].tier == "cold"
|
||||
|
||||
assert list(strategy.assemble([], budget)) == []
|
||||
print("phase3-temporal-ok")
|
||||
|
||||
|
||||
def _test_plan_decision_strategy() -> None:
|
||||
"""Test PlanDecisionContextStrategy warm/cold preference."""
|
||||
strategy = PlanDecisionContextStrategy()
|
||||
assert strategy.name == "plan-decision-context"
|
||||
assert strategy.can_handle({}) == 0.7
|
||||
|
||||
frags = [
|
||||
_frag("project://app/plan.py", "decision", 0.7, 20, 3, tier="warm"),
|
||||
_frag("project://app/new.py", "new code", 0.9, 15, 3, tier="hot"),
|
||||
]
|
||||
budget = ContextBudget(max_tokens=1000, reserved_tokens=0)
|
||||
result = list(strategy.assemble(frags, budget))
|
||||
assert len(result) > 0
|
||||
# Warm-tier should be first
|
||||
assert result[0].tier == "warm"
|
||||
|
||||
assert list(strategy.assemble([], budget)) == []
|
||||
print("phase3-plan-decision-ok")
|
||||
|
||||
|
||||
def _test_pipeline_integration() -> None:
|
||||
"""Test Phase 3 components injected into ACMSPipeline."""
|
||||
pipeline = ACMSPipeline(
|
||||
orderer=RelevanceCoherenceOrderer(),
|
||||
preamble_generator=ProvenancePreambleGenerator(),
|
||||
)
|
||||
frags = [
|
||||
_frag("project://app/main.py", "hello", 0.8, 10, 3),
|
||||
_frag("project://lib/util.py", "world", 0.6, 15, 5),
|
||||
]
|
||||
budget = ContextBudget(max_tokens=100000, reserved_tokens=0)
|
||||
payload = pipeline.assemble(
|
||||
plan_id=_PLAN_ID,
|
||||
fragments=frags,
|
||||
budget=budget,
|
||||
)
|
||||
assert len(payload.fragments) == 2
|
||||
assert payload.preamble is not None
|
||||
assert "Context Assembly Provenance" in payload.preamble
|
||||
print("phase3-pipeline-ok")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TESTS: dict[str, Callable[[], None]] = {
|
||||
"orderer-basic": _test_orderer_basic,
|
||||
"preamble-generate": _test_preamble_generate,
|
||||
"arce-strategy": _test_arce_strategy,
|
||||
"temporal-strategy": _test_temporal_strategy,
|
||||
"plan-decision-strategy": _test_plan_decision_strategy,
|
||||
"pipeline-integration": _test_pipeline_integration,
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in _TESTS:
|
||||
print(f"Usage: {sys.argv[0]} <{'|'.join(_TESTS)}>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
try:
|
||||
_TESTS[sys.argv[1]]()
|
||||
except Exception as exc:
|
||||
print(f"FAIL: {exc}", file=sys.stderr)
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -3,6 +3,11 @@
|
||||
Contains service classes that orchestrate business operations.
|
||||
"""
|
||||
|
||||
from cleveragents.application.services.acms_advanced_strategies import (
|
||||
ArceStrategy,
|
||||
PlanDecisionContextStrategy,
|
||||
TemporalArchaeologyStrategy,
|
||||
)
|
||||
from cleveragents.application.services.acms_phase2 import (
|
||||
ContentHashDeduplicator,
|
||||
GreedyKnapsackPacker,
|
||||
@@ -10,6 +15,10 @@ from cleveragents.application.services.acms_phase2 import (
|
||||
ScorerWeights,
|
||||
WeightedCompositeScorer,
|
||||
)
|
||||
from cleveragents.application.services.acms_phase3 import (
|
||||
ProvenancePreambleGenerator,
|
||||
RelevanceCoherenceOrderer,
|
||||
)
|
||||
from cleveragents.application.services.autonomy_controller import (
|
||||
AutonomyController,
|
||||
)
|
||||
@@ -166,6 +175,7 @@ __all__ = [
|
||||
"ApplyValidationGate",
|
||||
"ApplyValidationResult",
|
||||
"ApplyValidationSummary",
|
||||
"ArceStrategy",
|
||||
"AttachmentScope",
|
||||
"AutonomyController",
|
||||
"AutonomyGuardrailService",
|
||||
@@ -209,7 +219,10 @@ __all__ = [
|
||||
"PermissionService",
|
||||
"PersistentSessionService",
|
||||
"PipelineResultDict",
|
||||
"PlanDecisionContextStrategy",
|
||||
"PlanExecutionContext",
|
||||
"ProvenancePreambleGenerator",
|
||||
"RelevanceCoherenceOrderer",
|
||||
"ResolvedValue",
|
||||
"RuntimeExecuteActor",
|
||||
"RuntimeExecuteResult",
|
||||
@@ -241,6 +254,7 @@ __all__ = [
|
||||
"SubplanMergeService",
|
||||
"SubplanService",
|
||||
"SyntaxCheckRule",
|
||||
"TemporalArchaeologyStrategy",
|
||||
"ToolRegistryService",
|
||||
"TraceService",
|
||||
"UKOLoader",
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
"""Advanced built-in context strategies batch 2.
|
||||
|
||||
Implements the three advanced built-in context strategies from the spec
|
||||
(§25207-25216):
|
||||
|
||||
4. **ArceStrategy** (quality 0.95) — Adaptive Recursive Context Expansion.
|
||||
Multi-modal strategy combining all backends with iterative refinement.
|
||||
Highest quality strategy with configurable iteration limits.
|
||||
|
||||
5. **TemporalArchaeologyStrategy** (quality 0.5) — Historical context
|
||||
retrieval from cold storage. Queries graph+cold backends to discover
|
||||
temporal patterns in past decisions and archived context.
|
||||
|
||||
6. **PlanDecisionContextStrategy** (quality 0.7) — Decision history
|
||||
retrieval from warm/cold backends. Provides context from prior plan
|
||||
decisions and their outcomes for correction and retry scenarios.
|
||||
|
||||
All strategies implement the v1 ``ContextStrategy`` Protocol defined in
|
||||
``acms_service.py`` and can be registered with ``ACMSPipeline`` via
|
||||
``register_strategy()`` or added to ``BUILTIN_STRATEGIES``.
|
||||
|
||||
Based on ``docs/specification.md`` §25207-25216, §43183-43199.
|
||||
|
||||
ISSUES CLOSED: #545
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
|
||||
from cleveragents.application.services.acms_service import (
|
||||
StrategyCapabilities,
|
||||
_pack_budget,
|
||||
)
|
||||
from cleveragents.domain.models.core.context_fragment import (
|
||||
ContextBudget,
|
||||
ContextFragment,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ARCE iteration limit to prevent unbounded refinement (spec §43183-43191).
|
||||
DEFAULT_ARCE_MAX_ITERATIONS: Final[int] = 5
|
||||
|
||||
# ARCE convergence threshold: stop when score improvement is below this.
|
||||
DEFAULT_ARCE_CONVERGENCE_THRESHOLD: Final[float] = 0.01
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. ArceStrategy (quality 0.95)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ArceStrategy:
|
||||
"""Adaptive Recursive Context Expansion with iterative refinement.
|
||||
|
||||
The highest-quality built-in strategy, combining text, vector, and
|
||||
graph-based ranking with iterative multi-pass refinement.
|
||||
|
||||
**Algorithm**:
|
||||
|
||||
1. **Initial pass**: Score all fragments using a composite of keyword
|
||||
match, semantic similarity (word overlap), and hierarchy proximity.
|
||||
2. **Refinement loop**: Iteratively re-score fragments, boosting
|
||||
fragments that are contextually related to the current top-ranked
|
||||
set. Each iteration refines the ranking based on co-occurrence
|
||||
patterns in the selected subset.
|
||||
3. **Convergence check**: Stop when the ranking stabilises (score
|
||||
improvement below threshold) or ``max_iterations`` is reached.
|
||||
|
||||
The iteration limit (default 5) prevents unbounded refinement,
|
||||
satisfying the security requirement from the spec.
|
||||
|
||||
Implements ``ContextStrategy`` protocol from ``acms_service.py``.
|
||||
|
||||
Based on ``docs/specification.md`` §25214, §43183-43191 — ``arce``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_iterations: int = DEFAULT_ARCE_MAX_ITERATIONS,
|
||||
convergence_threshold: float = DEFAULT_ARCE_CONVERGENCE_THRESHOLD,
|
||||
) -> None:
|
||||
self._max_iterations = max_iterations
|
||||
self._convergence_threshold = convergence_threshold
|
||||
|
||||
@property
|
||||
def max_iterations(self) -> int:
|
||||
"""Return the configured maximum iteration limit."""
|
||||
return self._max_iterations
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "arce"
|
||||
|
||||
@property
|
||||
def capabilities(self) -> StrategyCapabilities:
|
||||
return StrategyCapabilities(
|
||||
supports_semantic_search=True,
|
||||
supports_graph_navigation=True,
|
||||
supports_temporal_archaeology=True,
|
||||
)
|
||||
|
||||
def can_handle(self, request: dict[str, Any]) -> float:
|
||||
"""Return 0.95 — always available as highest-quality strategy."""
|
||||
return 0.95
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
budget: ContextBudget,
|
||||
) -> Sequence[ContextFragment]:
|
||||
"""Rank fragments using iterative multi-backend refinement.
|
||||
|
||||
Performs multiple passes over the fragment set, progressively
|
||||
refining the ranking until convergence or the iteration limit
|
||||
is reached.
|
||||
"""
|
||||
if not fragments:
|
||||
return list(fragments)
|
||||
|
||||
# Initial scoring pass — composite of relevance, depth, and
|
||||
# keyword diversity
|
||||
scores: dict[str, float] = {}
|
||||
for frag in fragments:
|
||||
score = self._initial_score(frag)
|
||||
scores[frag.fragment_id] = score
|
||||
|
||||
# Iterative refinement loop
|
||||
iteration = 0
|
||||
prev_total = sum(scores.values())
|
||||
|
||||
while iteration < self._max_iterations:
|
||||
iteration += 1
|
||||
|
||||
# Refinement: boost fragments related to the current top set
|
||||
scores = self._refine_scores(fragments, scores)
|
||||
|
||||
# Check convergence
|
||||
current_total = sum(scores.values())
|
||||
improvement = abs(current_total - prev_total)
|
||||
if improvement < self._convergence_threshold:
|
||||
logger.info(
|
||||
"ARCE converged",
|
||||
extra={
|
||||
"iteration": iteration,
|
||||
"improvement": round(improvement, 6),
|
||||
},
|
||||
)
|
||||
break
|
||||
prev_total = current_total
|
||||
|
||||
logger.info(
|
||||
"ARCE refinement complete",
|
||||
extra={
|
||||
"iterations": iteration,
|
||||
"fragment_count": len(fragments),
|
||||
"max_iterations": self._max_iterations,
|
||||
},
|
||||
)
|
||||
|
||||
# Sort by refined score descending
|
||||
sorted_frags = sorted(
|
||||
fragments,
|
||||
key=lambda f: scores.get(f.fragment_id, 0.0),
|
||||
reverse=True,
|
||||
)
|
||||
return _pack_budget(sorted_frags, budget)
|
||||
|
||||
def explain(self) -> str:
|
||||
return (
|
||||
"Adaptive Recursive Context Expansion (ARCE). Multi-modal "
|
||||
"strategy combining text, vector, and graph-based ranking "
|
||||
f"with iterative refinement (max {self._max_iterations} "
|
||||
f"iterations, convergence threshold "
|
||||
f"{self._convergence_threshold}). Highest quality (0.95)."
|
||||
)
|
||||
|
||||
def _initial_score(self, frag: ContextFragment) -> float:
|
||||
"""Compute initial composite score for a fragment."""
|
||||
# Weighted combination: relevance (0.4), depth (0.3),
|
||||
# word diversity (0.3)
|
||||
depth_norm = frag.detail_depth / 9.0
|
||||
words = set(frag.content.lower().split())
|
||||
diversity = min(len(words) / max(frag.token_count, 1), 1.0)
|
||||
|
||||
return frag.relevance_score * 0.4 + depth_norm * 0.3 + diversity * 0.3
|
||||
|
||||
def _refine_scores(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
scores: dict[str, float],
|
||||
) -> dict[str, float]:
|
||||
"""Refine scores by boosting fragments related to the top set.
|
||||
|
||||
Identifies the top-ranked fragments and boosts others that share
|
||||
UKO node prefixes with the top set. This creates a feedback loop
|
||||
that progressively clusters contextually related content higher.
|
||||
"""
|
||||
if not fragments:
|
||||
return scores
|
||||
|
||||
# Identify top 30% as the "anchor" set
|
||||
sorted_ids = sorted(scores, key=lambda fid: scores[fid], reverse=True)
|
||||
anchor_count = max(1, len(sorted_ids) * 3 // 10)
|
||||
anchor_ids = set(sorted_ids[:anchor_count])
|
||||
|
||||
# Build anchor prefix set
|
||||
anchor_prefixes: set[str] = set()
|
||||
for frag in fragments:
|
||||
if frag.fragment_id in anchor_ids:
|
||||
prefix = _extract_node_prefix(frag.uko_node)
|
||||
anchor_prefixes.add(prefix)
|
||||
|
||||
# Refine scores
|
||||
refined: dict[str, float] = {}
|
||||
for frag in fragments:
|
||||
base_score = scores.get(frag.fragment_id, 0.0)
|
||||
prefix = _extract_node_prefix(frag.uko_node)
|
||||
|
||||
# Boost if related to anchor set
|
||||
if prefix in anchor_prefixes and frag.fragment_id not in anchor_ids:
|
||||
boost = 0.05 # Small contextual boost
|
||||
refined[frag.fragment_id] = min(base_score + boost, 1.0)
|
||||
else:
|
||||
refined[frag.fragment_id] = base_score
|
||||
|
||||
return refined
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. TemporalArchaeologyStrategy (quality 0.5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TemporalArchaeologyStrategy:
|
||||
"""Historical context retrieval from cold storage.
|
||||
|
||||
Searches for temporal patterns in past decisions and archived context.
|
||||
Prioritises fragments from cold storage tiers and older creation times,
|
||||
which represent historical decisions and archived knowledge.
|
||||
|
||||
In the v1 pipeline, this strategy operates on pre-fetched fragments
|
||||
rather than querying graph+cold backends directly. It simulates
|
||||
historical retrieval by preferring cold-tier fragments and older
|
||||
content, applying a recency-inverse scoring model.
|
||||
|
||||
Implements ``ContextStrategy`` protocol from ``acms_service.py``.
|
||||
|
||||
Based on ``docs/specification.md`` §25215, §43193-43195 —
|
||||
``temporal-archaeology``.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "temporal-archaeology"
|
||||
|
||||
@property
|
||||
def capabilities(self) -> StrategyCapabilities:
|
||||
return StrategyCapabilities(
|
||||
supports_temporal_archaeology=True,
|
||||
supports_graph_navigation=True,
|
||||
)
|
||||
|
||||
def can_handle(self, request: dict[str, Any]) -> float:
|
||||
"""Return 0.5 — moderate quality historical retrieval."""
|
||||
return 0.5
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
budget: ContextBudget,
|
||||
) -> Sequence[ContextFragment]:
|
||||
"""Rank fragments by historical relevance.
|
||||
|
||||
Prioritises cold-tier fragments and older creation times,
|
||||
representing archived historical context.
|
||||
"""
|
||||
if not fragments:
|
||||
return list(fragments)
|
||||
|
||||
# Score fragments: prefer cold tier and older timestamps
|
||||
scored: list[tuple[ContextFragment, float]] = []
|
||||
for frag in fragments:
|
||||
score = self._temporal_score(frag)
|
||||
scored.append((frag, score))
|
||||
|
||||
scored.sort(key=lambda pair: pair[1], reverse=True)
|
||||
sorted_frags = [frag for frag, _ in scored]
|
||||
|
||||
logger.info(
|
||||
"TemporalArchaeology ranked fragments",
|
||||
extra={
|
||||
"fragment_count": len(fragments),
|
||||
"cold_count": sum(1 for f in fragments if f.tier == "cold"),
|
||||
},
|
||||
)
|
||||
return _pack_budget(sorted_frags, budget)
|
||||
|
||||
def explain(self) -> str:
|
||||
return (
|
||||
"Historical context retrieval from cold storage and graph "
|
||||
"backends. Prioritises archived content and temporal patterns "
|
||||
"in past decisions. Quality 0.5."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _temporal_score(frag: ContextFragment) -> float:
|
||||
"""Compute a temporal relevance score.
|
||||
|
||||
Cold-tier fragments get a significant boost. Relevance score
|
||||
is used as a secondary factor.
|
||||
"""
|
||||
tier_bonus = {"cold": 0.4, "warm": 0.2, "hot": 0.0}
|
||||
tier_score = tier_bonus.get(frag.tier, 0.0)
|
||||
|
||||
return (
|
||||
tier_score * 0.5
|
||||
+ frag.relevance_score * 0.3
|
||||
+ (frag.detail_depth / 9.0) * 0.2
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. PlanDecisionContextStrategy (quality 0.7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TIER_PRIORITY_WARM_COLD: dict[str, int] = {"warm": 0, "cold": 1, "hot": 2}
|
||||
|
||||
|
||||
class PlanDecisionContextStrategy:
|
||||
"""Context from prior plan decisions and outcomes.
|
||||
|
||||
Retrieves fragments related to decision history, preferring warm
|
||||
and cold tier content that represents prior plan decisions and
|
||||
their outcomes. This is the primary strategy for correction and
|
||||
retry scenarios, where understanding what was previously decided
|
||||
(and why) is critical.
|
||||
|
||||
In the v1 pipeline, this strategy operates on pre-fetched fragments
|
||||
rather than querying warm/cold backends directly. It simulates
|
||||
decision-context retrieval by prioritising warm/cold fragments
|
||||
with higher relevance scores.
|
||||
|
||||
Implements ``ContextStrategy`` protocol from ``acms_service.py``.
|
||||
|
||||
Based on ``docs/specification.md`` §25216, §43197-43199 —
|
||||
``plan-decision-context``.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "plan-decision-context"
|
||||
|
||||
@property
|
||||
def capabilities(self) -> StrategyCapabilities:
|
||||
return StrategyCapabilities(
|
||||
supports_temporal_archaeology=True,
|
||||
)
|
||||
|
||||
def can_handle(self, request: dict[str, Any]) -> float:
|
||||
"""Return 0.7 — good quality decision context retrieval."""
|
||||
return 0.7
|
||||
|
||||
def assemble(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
budget: ContextBudget,
|
||||
) -> Sequence[ContextFragment]:
|
||||
"""Rank fragments by decision relevance from warm/cold tiers.
|
||||
|
||||
Prioritises warm and cold tier fragments, which represent prior
|
||||
plan decisions and outcomes.
|
||||
"""
|
||||
if not fragments:
|
||||
return list(fragments)
|
||||
|
||||
# Sort by tier priority (warm > cold > hot), then relevance
|
||||
sorted_frags = sorted(
|
||||
fragments,
|
||||
key=lambda f: (
|
||||
_TIER_PRIORITY_WARM_COLD.get(f.tier, 99),
|
||||
-f.relevance_score,
|
||||
),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"PlanDecisionContext ranked fragments",
|
||||
extra={
|
||||
"fragment_count": len(fragments),
|
||||
"warm_count": sum(1 for f in fragments if f.tier == "warm"),
|
||||
"cold_count": sum(1 for f in fragments if f.tier == "cold"),
|
||||
},
|
||||
)
|
||||
return _pack_budget(sorted_frags, budget)
|
||||
|
||||
def explain(self) -> str:
|
||||
return (
|
||||
"Retrieves context from prior plan decisions and their "
|
||||
"outcomes. Prioritises warm and cold tier fragments for "
|
||||
"correction and retry scenarios. Quality 0.7."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_node_prefix(uko_node: str) -> str:
|
||||
"""Extract the first two segments of a UKO node URI as a prefix."""
|
||||
if "://" in uko_node:
|
||||
uko_node = uko_node.split("://", 1)[1]
|
||||
segments = [seg for seg in uko_node.replace("\\", "/").split("/") if seg]
|
||||
return "/".join(segments[:2]) if len(segments) >= 2 else uko_node
|
||||
@@ -0,0 +1,315 @@
|
||||
"""ACMS pipeline Phase 3 — Context Finalization component implementations.
|
||||
|
||||
Provides the two production-grade Phase 3 components for the ACMS context
|
||||
assembly pipeline, replacing the no-op defaults from ``acms_service.py``:
|
||||
|
||||
8. **RelevanceCoherenceOrderer** — Orders budget-selected fragments for
|
||||
coherent presentation, balancing relevance ranking with narrative
|
||||
coherence. Groups related fragments (same UKO node prefix) together
|
||||
and maintains narrative flow within groups.
|
||||
|
||||
9. **ProvenancePreambleGenerator** — Generates a provenance preamble
|
||||
summarizing context assembly: which strategies contributed, confidence
|
||||
scores, coverage gaps, and fragment statistics.
|
||||
|
||||
All components satisfy the v1 Protocol signatures defined in
|
||||
``acms_service.py`` and can be injected into ``ACMSPipeline`` via
|
||||
constructor dependency injection.
|
||||
|
||||
Based on ``docs/specification.md`` §42648-42653.
|
||||
|
||||
ISSUES CLOSED: #545
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
|
||||
from cleveragents.domain.models.core.context_fragment import ContextFragment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Weight for relevance vs. coherence in the ordering algorithm.
|
||||
# Higher values favour relevance ranking; lower values favour grouping.
|
||||
DEFAULT_RELEVANCE_WEIGHT: Final[float] = 0.6
|
||||
DEFAULT_COHERENCE_WEIGHT: Final[float] = 0.4
|
||||
|
||||
# Minimum number of common URI path segments to consider two fragments
|
||||
# "related" for grouping purposes.
|
||||
DEFAULT_MIN_SHARED_SEGMENTS: Final[int] = 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 8. RelevanceCoherenceOrderer (FragmentOrderer protocol)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class RelevanceCoherenceOrderer:
|
||||
"""Order fragments by relevance while maintaining narrative coherence.
|
||||
|
||||
The ordering algorithm operates in two phases:
|
||||
|
||||
1. **Grouping phase**: Fragments are grouped by UKO node prefix
|
||||
(shared URI path segments). Related fragments that share a common
|
||||
ancestor in the knowledge hierarchy are placed together.
|
||||
|
||||
2. **Ranking phase**: Groups are sorted by the maximum relevance score
|
||||
of their members (descending). Within each group, fragments are
|
||||
sorted by relevance score descending then by detail depth ascending,
|
||||
so that higher-level overviews precede detailed content.
|
||||
|
||||
This produces a narrative-coherent ordering where related content
|
||||
is clustered together, but high-relevance content still appears first.
|
||||
|
||||
Implements ``FragmentOrderer`` protocol from ``acms_service.py``.
|
||||
|
||||
Based on ``docs/specification.md`` §42648 — ``FragmentOrdererProtocol``
|
||||
and ``RelevanceCoherenceOrderer``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
relevance_weight: float = DEFAULT_RELEVANCE_WEIGHT,
|
||||
coherence_weight: float = DEFAULT_COHERENCE_WEIGHT,
|
||||
min_shared_segments: int = DEFAULT_MIN_SHARED_SEGMENTS,
|
||||
) -> None:
|
||||
self._relevance_weight = relevance_weight
|
||||
self._coherence_weight = coherence_weight
|
||||
self._min_shared_segments = min_shared_segments
|
||||
|
||||
@property
|
||||
def relevance_weight(self) -> float:
|
||||
"""Return the configured relevance weight."""
|
||||
return self._relevance_weight
|
||||
|
||||
@property
|
||||
def coherence_weight(self) -> float:
|
||||
"""Return the configured coherence weight."""
|
||||
return self._coherence_weight
|
||||
|
||||
def order(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
) -> Sequence[ContextFragment]:
|
||||
"""Order fragments by relevance while maintaining narrative coherence.
|
||||
|
||||
Returns fragments grouped by related UKO node prefix, with groups
|
||||
ordered by maximum relevance and fragments within groups ordered
|
||||
by relevance descending then detail depth ascending.
|
||||
"""
|
||||
if not fragments or len(fragments) <= 1:
|
||||
return list(fragments)
|
||||
|
||||
# Phase 1: Group fragments by UKO node prefix
|
||||
groups = self._group_by_prefix(fragments)
|
||||
|
||||
# Phase 2: Sort groups by max relevance (descending)
|
||||
sorted_groups = sorted(
|
||||
groups.values(),
|
||||
key=lambda g: max(f.relevance_score for f in g),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Phase 3: Sort within each group by relevance desc, depth asc
|
||||
ordered: list[ContextFragment] = []
|
||||
for group in sorted_groups:
|
||||
sorted_group = sorted(
|
||||
group,
|
||||
key=lambda f: (-f.relevance_score, f.detail_depth),
|
||||
)
|
||||
ordered.extend(sorted_group)
|
||||
|
||||
logger.info(
|
||||
"Ordered fragments for coherent presentation",
|
||||
extra={
|
||||
"input_count": len(fragments),
|
||||
"group_count": len(sorted_groups),
|
||||
"output_count": len(ordered),
|
||||
},
|
||||
)
|
||||
return ordered
|
||||
|
||||
def _group_by_prefix(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
) -> dict[str, list[ContextFragment]]:
|
||||
"""Group fragments by shared UKO node prefix.
|
||||
|
||||
Fragments with at least ``min_shared_segments`` common URI path
|
||||
segments are placed in the same group. Fragments that don't
|
||||
match any group are placed in a singleton group.
|
||||
"""
|
||||
groups: dict[str, list[ContextFragment]] = defaultdict(list)
|
||||
|
||||
for frag in fragments:
|
||||
prefix = self._extract_prefix(frag.uko_node)
|
||||
groups[prefix].append(frag)
|
||||
|
||||
return dict(groups)
|
||||
|
||||
def _extract_prefix(self, uko_node: str) -> str:
|
||||
"""Extract the grouping prefix from a UKO node URI.
|
||||
|
||||
Takes the first ``min_shared_segments`` path segments as the
|
||||
prefix. If the URI has fewer segments, the full URI is used.
|
||||
"""
|
||||
segments = _uri_segments(uko_node)
|
||||
prefix_segments = segments[: self._min_shared_segments]
|
||||
return "/".join(prefix_segments) if prefix_segments else uko_node
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. ProvenancePreambleGenerator (PreambleGenerator protocol)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProvenancePreambleGenerator:
|
||||
"""Generate a provenance preamble summarizing context assembly.
|
||||
|
||||
Produces a structured text preamble that includes:
|
||||
|
||||
- **Strategy contributions**: Which strategies produced fragments and
|
||||
what percentage of the total context each contributed.
|
||||
- **Confidence indicators**: Average and range of relevance scores
|
||||
across all fragments.
|
||||
- **Coverage summary**: UKO node coverage, tier distribution, and
|
||||
depth distribution.
|
||||
- **Coverage gaps**: Identification of missing tiers or low-coverage
|
||||
areas.
|
||||
|
||||
The preamble is formatted as a concise, machine-readable text block
|
||||
suitable for prepending to the assembled context.
|
||||
|
||||
Implements ``PreambleGenerator`` protocol from ``acms_service.py``.
|
||||
|
||||
Based on ``docs/specification.md`` §42653 — ``PreambleGeneratorProtocol``
|
||||
and ``ProvenancePreambleGenerator``.
|
||||
"""
|
||||
|
||||
def generate(
|
||||
self,
|
||||
fragments: Sequence[ContextFragment],
|
||||
) -> str | None:
|
||||
"""Generate a structured provenance preamble.
|
||||
|
||||
Returns ``None`` if no fragments are provided.
|
||||
"""
|
||||
if not fragments:
|
||||
return None
|
||||
|
||||
total_tokens = sum(f.token_count for f in fragments)
|
||||
total_fragments = len(fragments)
|
||||
|
||||
# Strategy contributions
|
||||
strategy_counts: dict[str, int] = defaultdict(int)
|
||||
strategy_tokens: dict[str, int] = defaultdict(int)
|
||||
for frag in fragments:
|
||||
source = frag.strategy_source or frag.provenance.resource_type or "unknown"
|
||||
strategy_counts[source] += 1
|
||||
strategy_tokens[source] += frag.token_count
|
||||
|
||||
strategy_lines: list[str] = []
|
||||
for source in sorted(strategy_counts):
|
||||
count = strategy_counts[source]
|
||||
tokens = strategy_tokens[source]
|
||||
pct = (tokens / total_tokens * 100) if total_tokens > 0 else 0.0
|
||||
strategy_lines.append(
|
||||
f" - {source}: {count} fragments, {tokens} tokens ({pct:.1f}%)"
|
||||
)
|
||||
|
||||
# Confidence indicators
|
||||
scores = [f.relevance_score for f in fragments]
|
||||
avg_score = sum(scores) / len(scores)
|
||||
min_score = min(scores)
|
||||
max_score = max(scores)
|
||||
|
||||
# Tier distribution
|
||||
tier_counts: dict[str, int] = defaultdict(int)
|
||||
for frag in fragments:
|
||||
tier_counts[frag.tier] += 1
|
||||
|
||||
tier_lines: list[str] = []
|
||||
for tier in sorted(tier_counts):
|
||||
count = tier_counts[tier]
|
||||
tier_lines.append(f" - {tier}: {count} fragments")
|
||||
|
||||
# Depth distribution
|
||||
depth_counts: dict[int, int] = defaultdict(int)
|
||||
for frag in fragments:
|
||||
depth_counts[frag.detail_depth] += 1
|
||||
|
||||
depth_lines: list[str] = []
|
||||
for depth in sorted(depth_counts):
|
||||
count = depth_counts[depth]
|
||||
depth_lines.append(f" - depth {depth}: {count} fragments")
|
||||
|
||||
# UKO node coverage
|
||||
unique_nodes = len({f.uko_node for f in fragments})
|
||||
|
||||
# Coverage gaps
|
||||
gaps: list[str] = []
|
||||
all_tiers = {"hot", "warm", "cold"}
|
||||
missing_tiers = all_tiers - set(tier_counts)
|
||||
if missing_tiers:
|
||||
gaps.append(f" - Missing tiers: {', '.join(sorted(missing_tiers))}")
|
||||
if avg_score < 0.3:
|
||||
gaps.append(f" - Low average confidence: {avg_score:.3f}")
|
||||
if total_fragments < 3:
|
||||
gaps.append(f" - Low fragment count: {total_fragments}")
|
||||
|
||||
# Build preamble
|
||||
lines = [
|
||||
"--- Context Assembly Provenance ---",
|
||||
f"Fragments: {total_fragments} | Tokens: {total_tokens} "
|
||||
f"| UKO Nodes: {unique_nodes}",
|
||||
"",
|
||||
"Strategy Contributions:",
|
||||
*strategy_lines,
|
||||
"",
|
||||
f"Confidence: avg={avg_score:.3f} min={min_score:.3f} max={max_score:.3f}",
|
||||
"",
|
||||
"Tier Distribution:",
|
||||
*tier_lines,
|
||||
"",
|
||||
"Depth Distribution:",
|
||||
*depth_lines,
|
||||
]
|
||||
|
||||
if gaps:
|
||||
lines.extend(["", "Coverage Gaps:", *gaps])
|
||||
|
||||
lines.append("--- End Provenance ---")
|
||||
|
||||
preamble = "\n".join(lines)
|
||||
|
||||
logger.info(
|
||||
"Generated provenance preamble",
|
||||
extra={
|
||||
"fragment_count": total_fragments,
|
||||
"total_tokens": total_tokens,
|
||||
"strategy_count": len(strategy_counts),
|
||||
"unique_nodes": unique_nodes,
|
||||
"gap_count": len(gaps),
|
||||
},
|
||||
)
|
||||
return preamble
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _uri_segments(uri: str) -> list[str]:
|
||||
"""Split a UKO URI into path segments for hierarchy comparison."""
|
||||
if "://" in uri:
|
||||
uri = uri.split("://", 1)[1]
|
||||
return [seg for seg in uri.replace("\\", "/").split("/") if seg]
|
||||
@@ -856,3 +856,17 @@ LAYER1_CODE_CLASSES # noqa: B018, F821
|
||||
LAYER1_DOC_CLASSES # noqa: B018, F821
|
||||
LAYER1_DATA_CLASSES # noqa: B018, F821
|
||||
LAYER1_INFRA_CLASSES # noqa: B018, F821
|
||||
|
||||
# ACMS Pipeline Phase 3 — Context Finalization components (public API, issue #545)
|
||||
RelevanceCoherenceOrderer # noqa: B018, F821
|
||||
ProvenancePreambleGenerator # noqa: B018, F821
|
||||
relevance_weight # noqa: B018, F821
|
||||
coherence_weight # noqa: B018, F821
|
||||
|
||||
# ACMS Advanced Strategies — batch 2 (public API, issue #545)
|
||||
ArceStrategy # noqa: B018, F821
|
||||
TemporalArchaeologyStrategy # noqa: B018, F821
|
||||
PlanDecisionContextStrategy # noqa: B018, F821
|
||||
max_iterations # noqa: B018, F821
|
||||
_temporal_score # noqa: B018, F821
|
||||
_extract_node_prefix # noqa: B018, F821
|
||||
|
||||
Reference in New Issue
Block a user