diff --git a/benchmarks/context_fragment_models_bench.py b/benchmarks/context_fragment_models_bench.py new file mode 100644 index 000000000..ec5a3fded --- /dev/null +++ b/benchmarks/context_fragment_models_bench.py @@ -0,0 +1,152 @@ +"""ASV benchmarks for ScoredFragment and ContextFragment pipeline models. + +Measures the performance of: +- ScoredFragment creation (with and without breakdown) +- ScoredFragment factory methods +- ScoredFragment equality comparison +- ScoredFragment set-based deduplication +- ContextFragment with strategy_source +""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +import cleveragents # noqa: E402 + +importlib.reload(cleveragents) + +from cleveragents.domain.contexts.fragment import ScoredFragment # noqa: E402 +from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 + ContextFragment, + FragmentProvenance, +) + +_DEFAULT_PROV = FragmentProvenance(resource_uri="bench://default") + + +def _make_fragment(**kwargs: object) -> ContextFragment: + """Create a ContextFragment with sensible benchmark defaults.""" + defaults: dict[str, object] = { + "uko_node": "bench://file", + "content": "benchmark content", + "token_count": 10, + "provenance": _DEFAULT_PROV, + } + defaults.update(kwargs) + return ContextFragment(**defaults) # type: ignore[arg-type] + + +class ScoredFragmentCreationSuite: + """Benchmark ScoredFragment creation throughput.""" + + def setup(self) -> None: + """Pre-create fragment for repeated scored-fragment construction.""" + self._fragment = _make_fragment() + self._fragment_with_relevance = _make_fragment(relevance_score=0.85) + self._breakdown = { + "relevance": 0.9, + "hierarchy": 0.8, + "recency": 0.95, + } + + def time_scored_fragment_minimal(self) -> None: + """Benchmark creating a ScoredFragment with minimal fields.""" + ScoredFragment( + fragment=self._fragment, + composite_score=0.85, + ) + + def time_scored_fragment_with_breakdown(self) -> None: + """Benchmark creating a ScoredFragment with full breakdown.""" + ScoredFragment( + fragment=self._fragment, + composite_score=0.92, + score_breakdown=self._breakdown, + rank=1, + ) + + def time_scored_fragment_from_relevance(self) -> None: + """Benchmark ScoredFragment.from_relevance factory.""" + ScoredFragment.from_relevance(self._fragment_with_relevance) + + def time_scored_fragment_from_factory(self) -> None: + """Benchmark ScoredFragment.from_fragment factory.""" + ScoredFragment.from_fragment( + self._fragment, + composite_score=0.6, + score_breakdown=self._breakdown, + rank=3, + ) + + +class ScoredFragmentEqualitySuite: + """Benchmark ScoredFragment equality and hashing.""" + + def setup(self) -> None: + """Pre-create scored fragments for comparison benchmarks.""" + frag_a = _make_fragment( + uko_node="bench://file/a", + content="A", + detail_depth=3, + ) + frag_b = _make_fragment( + uko_node="bench://file/a", + content="B", + detail_depth=3, + ) + self._scored_a = ScoredFragment.from_fragment(frag_a, composite_score=0.5) + self._scored_b = ScoredFragment.from_fragment(frag_b, composite_score=0.9) + # Build a list of 100 fragments (50 unique uko_node values) + self._scored_list = [ + ScoredFragment.from_fragment( + _make_fragment( + uko_node=f"bench://file/{i % 50}", + content=f"content {i}", + detail_depth=3, + ), + composite_score=round((i % 10) / 10, 1), + ) + for i in range(100) + ] + + def time_equality_check(self) -> None: + """Benchmark equality comparison.""" + _ = self._scored_a == self._scored_b + + def time_hash_computation(self) -> None: + """Benchmark hash computation.""" + hash(self._scored_a) + + def time_set_deduplication_100(self) -> None: + """Benchmark set-based deduplication of 100 scored fragments.""" + set(self._scored_list) + + +class ContextFragmentStrategySuite: + """Benchmark ContextFragment with strategy_source field.""" + + def time_fragment_with_strategy_source(self) -> None: + """Benchmark creating a ContextFragment with strategy_source.""" + ContextFragment( + uko_node="bench://file", + content="benchmark content", + token_count=10, + provenance=_DEFAULT_PROV, + strategy_source="keyword-search", + ) + + def time_fragment_default_strategy_source(self) -> None: + """Benchmark creating a ContextFragment with default strategy_source.""" + ContextFragment( + uko_node="bench://file", + content="benchmark content", + token_count=10, + provenance=_DEFAULT_PROV, + ) diff --git a/features/context_fragment_models.feature b/features/context_fragment_models.feature new file mode 100644 index 000000000..253b21003 --- /dev/null +++ b/features/context_fragment_models.feature @@ -0,0 +1,136 @@ +@phase2 @acms @context_fragment_models +Feature: ScoredFragment and ContextFragment pipeline models + As a CleverAgents developer + I want pipeline-specific fragment models with scoring and deduplication + So that the ACMS context assembly pipeline can rank, deduplicate, and pack fragments + + # --------------------------------------------------------------------------- + # ScoredFragment — Construction + # --------------------------------------------------------------------------- + + @scored_fragment + Scenario: Create a ScoredFragment with required fields + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with composite_score 0.85 + Then the scored fragment composite_score should be 0.85 + And the scored fragment rank should be 0 + And the scored fragment score_breakdown should be empty + And the scored fragment uko_node should be "project://app/main.py" + And the scored fragment detail_depth should be 0 + And the scored fragment token_count should be 10 + + @scored_fragment + Scenario: Create a ScoredFragment with all fields + Given a context fragment with uko_node "project://app/io.py" and content "async IO" and token_count 50 + When I create a scored fragment with composite_score 0.92 and rank 1 and breakdown: + | component | score | + | relevance | 0.9 | + | hierarchy | 0.8 | + | recency | 0.95 | + Then the scored fragment composite_score should be 0.92 + And the scored fragment rank should be 1 + And the scored fragment breakdown "relevance" should be 0.9 + And the scored fragment breakdown "hierarchy" should be 0.8 + And the scored fragment breakdown "recency" should be 0.95 + + @scored_fragment + Scenario: Create a ScoredFragment from relevance factory + Given a context fragment with uko_node "project://app/util.py" and content "utils" and token_count 20 and relevance 0.75 + When I create a scored fragment from relevance + Then the scored fragment composite_score should be 0.75 + And the scored fragment breakdown "relevance" should be 0.75 + And the scored fragment rank should be 0 + + @scored_fragment + Scenario: Create a ScoredFragment from fragment factory + Given a context fragment with uko_node "project://app/core.py" and content "core" and token_count 30 + When I create a scored fragment from factory with score 0.6 and rank 3 + Then the scored fragment composite_score should be 0.6 + And the scored fragment rank should be 3 + + # --------------------------------------------------------------------------- + # ScoredFragment — Equality and Hashing + # --------------------------------------------------------------------------- + + @scored_fragment @equality + Scenario: ScoredFragments with same uko_node and detail_depth are equal + Given two scored fragments with uko_node "project://app/main.py" and detail_depth 3 + Then the two scored fragments should be equal + And the two scored fragments should have the same hash + + @scored_fragment @equality + Scenario: ScoredFragments with different uko_node are not equal + Given a scored fragment with uko_node "project://app/main.py" and detail_depth 3 + And another scored fragment with uko_node "project://app/other.py" and detail_depth 3 + Then the two scored fragments should not be equal + + @scored_fragment @equality + Scenario: ScoredFragments with different detail_depth are not equal + Given a scored fragment with uko_node "project://app/main.py" and detail_depth 3 + And another scored fragment with uko_node "project://app/main.py" and detail_depth 5 + Then the two scored fragments should not be equal + + @scored_fragment @equality + Scenario: ScoredFragments can be deduplicated via set + Given three scored fragments where two share uko_node "project://app/main.py" detail_depth 3 + When I add them to a set + Then the set should contain 2 unique scored fragments + + @scored_fragment @equality + Scenario: ScoredFragments with same identity but different scores are equal + Given a scored fragment with uko_node "project://app/main.py" and detail_depth 2 and score 0.5 + And another scored fragment with uko_node "project://app/main.py" and detail_depth 2 and score 0.9 + Then the two scored fragments should be equal + And the two scored fragments should have the same hash + + # --------------------------------------------------------------------------- + # ScoredFragment — Immutability + # --------------------------------------------------------------------------- + + @scored_fragment @immutability + Scenario: ScoredFragment is frozen + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with composite_score 0.85 + Then modifying the scored fragment composite_score should raise an error + + # --------------------------------------------------------------------------- + # ScoredFragment — Validation + # --------------------------------------------------------------------------- + + @scored_fragment @validation + Scenario: Invalid composite_score above 1.0 rejected + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with composite_score 1.5 + Then a validation error should be raised + + @scored_fragment @validation + Scenario: Invalid composite_score below 0.0 rejected + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with composite_score -0.1 + Then a validation error should be raised + + @scored_fragment @validation + Scenario: Invalid score_breakdown component rejected + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with an invalid breakdown component 1.5 + Then a validation error should be raised + + @scored_fragment @validation + Scenario: Negative rank rejected + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + When I create a scored fragment with composite_score 0.5 and rank -1 + Then a validation error should be raised + + # --------------------------------------------------------------------------- + # ContextFragment — strategy_source field + # --------------------------------------------------------------------------- + + @context_fragment @strategy_source + Scenario: ContextFragment has strategy_source field with default + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 + Then the fragment strategy_source should be empty + + @context_fragment @strategy_source + Scenario: ContextFragment strategy_source can be set + Given a context fragment with uko_node "project://app/main.py" and content "hello" and token_count 10 and strategy_source "keyword-search" + Then the fragment strategy_source should be "keyword-search" diff --git a/features/steps/context_fragment_models_steps.py b/features/steps/context_fragment_models_steps.py new file mode 100644 index 000000000..83ac568e3 --- /dev/null +++ b/features/steps/context_fragment_models_steps.py @@ -0,0 +1,405 @@ +"""Step definitions for features/context_fragment_models.feature. + +Tests the ScoredFragment pipeline model and ContextFragment strategy_source +field directly in-memory -- no database required. +""" + +from __future__ import annotations + +from typing import Any + +from behave import given, then, when +from behave.runner import Context +from pydantic import ValidationError + +from cleveragents.domain.contexts.fragment import ScoredFragment +from cleveragents.domain.models.core.context_fragment import ( + ContextFragment, + FragmentProvenance, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_DEFAULT_PROVENANCE = FragmentProvenance(resource_uri="test://default") + + +def _make_fragment(**kwargs: Any) -> ContextFragment: + """Create a ContextFragment with sensible test defaults.""" + kwargs.setdefault("uko_node", "test://default") + kwargs.setdefault("content", "test content") + kwargs.setdefault("token_count", 0) + kwargs.setdefault("provenance", _DEFAULT_PROVENANCE) + return ContextFragment(**kwargs) + + +# --------------------------------------------------------------------------- +# Given steps +# --------------------------------------------------------------------------- + + +@given( + 'a context fragment with uko_node "{uko}" and content "{content}"' + " and token_count {tokens:d} and relevance {rel:g}" +) +def step_given_fragment_with_relevance( + context: Context, + uko: str, + content: str, + tokens: int, + rel: float, +) -> None: + context.fragment = _make_fragment( + uko_node=uko, + content=content, + token_count=tokens, + relevance_score=rel, + ) + + +@given( + 'a context fragment with uko_node "{uko}" and content "{content}"' + ' and token_count {tokens:d} and strategy_source "{strategy}"' +) +def step_given_fragment_with_strategy( + context: Context, + uko: str, + content: str, + tokens: int, + strategy: str, +) -> None: + context.fragment = _make_fragment( + uko_node=uko, + content=content, + token_count=tokens, + strategy_source=strategy, + ) + + +@given('two scored fragments with uko_node "{uko}" and detail_depth {depth:d}') +def step_given_two_same_scored_fragments( + context: Context, + uko: str, + depth: int, +) -> None: + frag_a = _make_fragment( + uko_node=uko, + content="content A", + detail_depth=depth, + token_count=10, + ) + frag_b = _make_fragment( + uko_node=uko, + content="content B", + detail_depth=depth, + token_count=20, + ) + context.scored_a = ScoredFragment.from_fragment(frag_a, composite_score=0.5) + context.scored_b = ScoredFragment.from_fragment(frag_b, composite_score=0.9) + + +@given('a scored fragment with uko_node "{uko}" and detail_depth {depth:d}') +def step_given_scored_fragment_first( + context: Context, + uko: str, + depth: int, +) -> None: + frag = _make_fragment( + uko_node=uko, + content="content", + detail_depth=depth, + token_count=10, + ) + context.scored_a = ScoredFragment.from_fragment(frag, composite_score=0.5) + + +@given('another scored fragment with uko_node "{uko}" and detail_depth {depth:d}') +def step_given_scored_fragment_second( + context: Context, + uko: str, + depth: int, +) -> None: + frag = _make_fragment( + uko_node=uko, + content="content", + detail_depth=depth, + token_count=10, + ) + context.scored_b = ScoredFragment.from_fragment(frag, composite_score=0.7) + + +@given( + 'a scored fragment with uko_node "{uko}" and detail_depth {depth:d}' + " and score {score:g}" +) +def step_given_scored_fragment_with_score( + context: Context, + uko: str, + depth: int, + score: float, +) -> None: + frag = _make_fragment( + uko_node=uko, + content="content", + detail_depth=depth, + token_count=10, + ) + context.scored_a = ScoredFragment.from_fragment(frag, composite_score=score) + + +@given( + 'another scored fragment with uko_node "{uko}" and detail_depth {depth:d}' + " and score {score:g}" +) +def step_given_another_scored_fragment_with_score( + context: Context, + uko: str, + depth: int, + score: float, +) -> None: + frag = _make_fragment( + uko_node=uko, + content="content", + detail_depth=depth, + token_count=10, + ) + context.scored_b = ScoredFragment.from_fragment(frag, composite_score=score) + + +@given('three scored fragments where two share uko_node "{uko}" detail_depth {depth:d}') +def step_given_three_scored_fragments( + context: Context, + uko: str, + depth: int, +) -> None: + frag_a = _make_fragment( + uko_node=uko, + content="content A", + detail_depth=depth, + token_count=10, + ) + frag_b = _make_fragment( + uko_node=uko, + content="content B", + detail_depth=depth, + token_count=20, + ) + frag_c = _make_fragment( + uko_node="project://app/other.py", + content="content C", + detail_depth=depth, + token_count=30, + ) + context.scored_list = [ + ScoredFragment.from_fragment(frag_a, composite_score=0.5), + ScoredFragment.from_fragment(frag_b, composite_score=0.9), + ScoredFragment.from_fragment(frag_c, composite_score=0.7), + ] + + +# --------------------------------------------------------------------------- +# When steps +# --------------------------------------------------------------------------- + + +@when("I create a scored fragment with composite_score {score:g}") +def step_when_create_scored(context: Context, score: float) -> None: + try: + context.scored = ScoredFragment( + fragment=context.fragment, + composite_score=score, + ) + context.error = None + except (ValidationError, ValueError) as exc: + context.scored = None + context.error = exc + + +@when( + "I create a scored fragment with composite_score {score:g}" + " and rank {rank:d} and breakdown:" +) +def step_when_create_scored_with_breakdown( + context: Context, + score: float, + rank: int, +) -> None: + breakdown: dict[str, float] = {} + assert context.table is not None + for row in context.table: + breakdown[row["component"]] = float(row["score"]) + context.scored = ScoredFragment( + fragment=context.fragment, + composite_score=score, + score_breakdown=breakdown, + rank=rank, + ) + + +@when("I create a scored fragment from relevance") +def step_when_create_from_relevance(context: Context) -> None: + context.scored = ScoredFragment.from_relevance(context.fragment) + + +@when("I create a scored fragment from factory with score {score:g} and rank {rank:d}") +def step_when_create_from_factory( + context: Context, + score: float, + rank: int, +) -> None: + context.scored = ScoredFragment.from_fragment( + context.fragment, + composite_score=score, + rank=rank, + ) + + +@when("I create a scored fragment with an invalid breakdown component {score:g}") +def step_when_create_invalid_breakdown(context: Context, score: float) -> None: + try: + context.scored = ScoredFragment( + fragment=context.fragment, + composite_score=0.5, + score_breakdown={"bad": score}, + ) + context.error = None + except (ValidationError, ValueError) as exc: + context.scored = None + context.error = exc + + +@when("I create a scored fragment with composite_score {score:g} and rank {rank:d}") +def step_when_create_scored_with_rank( + context: Context, + score: float, + rank: int, +) -> None: + try: + context.scored = ScoredFragment( + fragment=context.fragment, + composite_score=score, + rank=rank, + ) + context.error = None + except (ValidationError, ValueError) as exc: + context.scored = None + context.error = exc + + +@when("I add them to a set") +def step_when_add_to_set(context: Context) -> None: + context.scored_set = set(context.scored_list) + + +# --------------------------------------------------------------------------- +# Then steps +# --------------------------------------------------------------------------- + + +@then("the scored fragment composite_score should be {expected:g}") +def step_then_composite_score(context: Context, expected: float) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.composite_score == expected, ( + f"Expected composite_score={expected}, got {context.scored.composite_score}" + ) + + +@then("the scored fragment rank should be {expected:d}") +def step_then_rank(context: Context, expected: int) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.rank == expected, ( + f"Expected rank={expected}, got {context.scored.rank}" + ) + + +@then("the scored fragment score_breakdown should be empty") +def step_then_breakdown_empty(context: Context) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.score_breakdown == {}, ( + f"Expected empty breakdown, got {context.scored.score_breakdown}" + ) + + +@then('the scored fragment uko_node should be "{expected}"') +def step_then_scored_uko_node(context: Context, expected: str) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.uko_node == expected, ( + f"Expected uko_node={expected}, got {context.scored.uko_node}" + ) + + +@then("the scored fragment detail_depth should be {expected:d}") +def step_then_scored_detail_depth(context: Context, expected: int) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.detail_depth == expected, ( + f"Expected detail_depth={expected}, got {context.scored.detail_depth}" + ) + + +@then("the scored fragment token_count should be {expected:d}") +def step_then_scored_token_count(context: Context, expected: int) -> None: + assert context.scored is not None, "No scored fragment created" + assert context.scored.token_count == expected, ( + f"Expected token_count={expected}, got {context.scored.token_count}" + ) + + +@then('the scored fragment breakdown "{key}" should be {expected:g}') +def step_then_breakdown_value(context: Context, key: str, expected: float) -> None: + assert context.scored is not None, "No scored fragment created" + assert key in context.scored.score_breakdown, ( + f"Key '{key}' not in score_breakdown: {context.scored.score_breakdown}" + ) + actual = context.scored.score_breakdown[key] + assert actual == expected, f"Expected breakdown['{key}']={expected}, got {actual}" + + +@then("the two scored fragments should be equal") +def step_then_scored_equal(context: Context) -> None: + assert context.scored_a == context.scored_b, "Expected scored fragments to be equal" + + +@then("the two scored fragments should not be equal") +def step_then_scored_not_equal(context: Context) -> None: + assert context.scored_a != context.scored_b, ( + "Expected scored fragments to NOT be equal" + ) + + +@then("the two scored fragments should have the same hash") +def step_then_scored_same_hash(context: Context) -> None: + assert hash(context.scored_a) == hash(context.scored_b), ( + "Expected scored fragments to have the same hash" + ) + + +@then("the set should contain {expected:d} unique scored fragments") +def step_then_set_size(context: Context, expected: int) -> None: + assert len(context.scored_set) == expected, ( + f"Expected {expected} unique fragments, got {len(context.scored_set)}" + ) + + +@then("modifying the scored fragment composite_score should raise an error") +def step_then_frozen(context: Context) -> None: + try: + context.scored.composite_score = 0.99 # type: ignore[misc] + raise AssertionError("Expected ValidationError for frozen model") + except ValidationError: + pass + + +@then("the fragment strategy_source should be empty") +def step_then_strategy_source_empty(context: Context) -> None: + assert context.fragment.strategy_source == "", ( + f"Expected empty strategy_source, got '{context.fragment.strategy_source}'" + ) + + +@then('the fragment strategy_source should be "{expected}"') +def step_then_strategy_source(context: Context, expected: str) -> None: + assert context.fragment.strategy_source == expected, ( + f"Expected strategy_source='{expected}', " + f"got '{context.fragment.strategy_source}'" + ) diff --git a/robot/context_fragment_models.robot b/robot/context_fragment_models.robot new file mode 100644 index 000000000..a56e4a859 --- /dev/null +++ b/robot/context_fragment_models.robot @@ -0,0 +1,65 @@ +*** Settings *** +Documentation Integration smoke tests for ScoredFragment and ContextFragment pipeline models +Resource ${CURDIR}/common.resource +Suite Setup Setup Test Environment +Suite Teardown Cleanup Test Environment + +*** Variables *** +${HELPER} ${CURDIR}/helper_context_fragment_models.py + +*** Test Cases *** +Create ScoredFragment With Defaults + [Documentation] Create a ScoredFragment and verify default fields + ${result}= Run Process ${PYTHON} ${HELPER} scored-create cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-create-ok + +Create ScoredFragment With Breakdown + [Documentation] Create a ScoredFragment with full score breakdown + ${result}= Run Process ${PYTHON} ${HELPER} scored-breakdown cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-breakdown-ok + +ScoredFragment Equality By Identity + [Documentation] Verify equality based on uko_node + detail_depth + ${result}= Run Process ${PYTHON} ${HELPER} scored-equality cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-equality-ok + +ScoredFragment Deduplication Via Set + [Documentation] Verify set-based deduplication works + ${result}= Run Process ${PYTHON} ${HELPER} scored-dedup cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-dedup-ok + +ScoredFragment From Relevance Factory + [Documentation] Create ScoredFragment using from_relevance factory + ${result}= Run Process ${PYTHON} ${HELPER} scored-from-relevance cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-from-relevance-ok + +ScoredFragment Immutability + [Documentation] Verify ScoredFragment is frozen + ${result}= Run Process ${PYTHON} ${HELPER} scored-frozen cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} scored-frozen-ok + +ContextFragment Strategy Source + [Documentation] Verify strategy_source field on ContextFragment + ${result}= Run Process ${PYTHON} ${HELPER} strategy-source cwd=${WORKSPACE} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Contain ${result.stdout} strategy-source-ok diff --git a/robot/helper_context_fragment_models.py b/robot/helper_context_fragment_models.py new file mode 100644 index 000000000..3e82e7001 --- /dev/null +++ b/robot/helper_context_fragment_models.py @@ -0,0 +1,227 @@ +"""Robot Framework helper for ScoredFragment and ContextFragment integration tests. + +Provides a CLI-style interface for Robot to invoke pipeline model +operations and verify results. Exit code 0 = success, 1 = failure. + +Usage: + python robot/helper_context_fragment_models.py scored-create + python robot/helper_context_fragment_models.py scored-breakdown + python robot/helper_context_fragment_models.py scored-equality + python robot/helper_context_fragment_models.py scored-dedup + python robot/helper_context_fragment_models.py scored-from-relevance + python robot/helper_context_fragment_models.py scored-frozen + python robot/helper_context_fragment_models.py strategy-source +""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + +from pydantic import ValidationError + +_SRC = str(Path(__file__).resolve().parents[1] / "src") +if _SRC not in sys.path: + sys.path.insert(0, _SRC) + +from cleveragents.domain.contexts.fragment import ScoredFragment # noqa: E402 +from cleveragents.domain.models.core.context_fragment import ( # noqa: E402 + ContextFragment, + FragmentProvenance, +) + +_DEFAULT_PROV = FragmentProvenance(resource_uri="test://robot") + + +def _make_fragment(**kwargs: object) -> ContextFragment: + """Create a ContextFragment with sensible test defaults.""" + defaults: dict[str, object] = { + "uko_node": "test://default", + "content": "test content", + "token_count": 10, + "provenance": _DEFAULT_PROV, + } + defaults.update(kwargs) + return ContextFragment(**defaults) # type: ignore[arg-type] + + +def _cmd_scored_create() -> int: + """Create a ScoredFragment and verify defaults.""" + frag = _make_fragment(uko_node="project://app/main.py", content="hello") + scored = ScoredFragment(fragment=frag, composite_score=0.85) + if scored.composite_score != 0.85: + print(f"fail: expected composite_score=0.85, got {scored.composite_score}") + return 1 + if scored.rank != 0: + print(f"fail: expected rank=0, got {scored.rank}") + return 1 + if scored.score_breakdown != {}: + print(f"fail: expected empty breakdown, got {scored.score_breakdown}") + return 1 + if scored.uko_node != "project://app/main.py": + print(f"fail: expected uko_node=project://app/main.py, got {scored.uko_node}") + return 1 + print("scored-create-ok") + return 0 + + +def _cmd_scored_breakdown() -> int: + """Create a ScoredFragment with full breakdown.""" + frag = _make_fragment(uko_node="project://app/io.py", content="async IO") + breakdown = {"relevance": 0.9, "hierarchy": 0.8, "recency": 0.95} + scored = ScoredFragment( + fragment=frag, + composite_score=0.92, + score_breakdown=breakdown, + rank=1, + ) + if scored.composite_score != 0.92: + print(f"fail: expected composite_score=0.92, got {scored.composite_score}") + return 1 + if scored.rank != 1: + print(f"fail: expected rank=1, got {scored.rank}") + return 1 + if scored.score_breakdown.get("relevance") != 0.9: + print(f"fail: expected relevance=0.9, got {scored.score_breakdown}") + return 1 + print("scored-breakdown-ok") + return 0 + + +def _cmd_scored_equality() -> int: + """Verify equality based on uko_node + detail_depth.""" + frag_a = _make_fragment( + uko_node="project://app/main.py", + content="A", + detail_depth=3, + ) + frag_b = _make_fragment( + uko_node="project://app/main.py", + content="B", + detail_depth=3, + ) + sa = ScoredFragment.from_fragment(frag_a, composite_score=0.5) + sb = ScoredFragment.from_fragment(frag_b, composite_score=0.9) + if sa != sb: + print("fail: expected equal scored fragments") + return 1 + if hash(sa) != hash(sb): + print("fail: expected same hash") + return 1 + # Different uko_node should be not equal + frag_c = _make_fragment( + uko_node="project://app/other.py", + content="C", + detail_depth=3, + ) + sc = ScoredFragment.from_fragment(frag_c, composite_score=0.5) + if sa == sc: + print("fail: expected different scored fragments") + return 1 + print("scored-equality-ok") + return 0 + + +def _cmd_scored_dedup() -> int: + """Verify set-based deduplication.""" + frag_a = _make_fragment( + uko_node="project://app/main.py", + content="A", + detail_depth=3, + ) + frag_b = _make_fragment( + uko_node="project://app/main.py", + content="B", + detail_depth=3, + ) + frag_c = _make_fragment( + uko_node="project://app/other.py", + content="C", + detail_depth=3, + ) + scored_set = { + ScoredFragment.from_fragment(frag_a, composite_score=0.5), + ScoredFragment.from_fragment(frag_b, composite_score=0.9), + ScoredFragment.from_fragment(frag_c, composite_score=0.7), + } + if len(scored_set) != 2: + print(f"fail: expected 2 unique, got {len(scored_set)}") + return 1 + print("scored-dedup-ok") + return 0 + + +def _cmd_scored_from_relevance() -> int: + """Create ScoredFragment using from_relevance factory.""" + frag = _make_fragment( + uko_node="project://app/util.py", + content="utils", + relevance_score=0.75, + ) + scored = ScoredFragment.from_relevance(frag) + if scored.composite_score != 0.75: + print(f"fail: expected composite_score=0.75, got {scored.composite_score}") + return 1 + if scored.score_breakdown.get("relevance") != 0.75: + print(f"fail: expected relevance=0.75, got {scored.score_breakdown}") + return 1 + print("scored-from-relevance-ok") + return 0 + + +def _cmd_scored_frozen() -> int: + """Verify ScoredFragment is frozen.""" + frag = _make_fragment() + scored = ScoredFragment(fragment=frag, composite_score=0.5) + try: + scored.composite_score = 0.99 # type: ignore[misc] + print("fail: expected error when modifying frozen model") + return 1 + except ValidationError: + pass + print("scored-frozen-ok") + return 0 + + +def _cmd_strategy_source() -> int: + """Verify strategy_source field on ContextFragment.""" + # Default should be empty string + frag = _make_fragment() + if frag.strategy_source != "": + print(f"fail: expected empty strategy_source, got '{frag.strategy_source}'") + return 1 + # Explicit value + frag2 = _make_fragment(strategy_source="keyword-search") + if frag2.strategy_source != "keyword-search": + print( + f"fail: expected strategy_source='keyword-search', " + f"got '{frag2.strategy_source}'" + ) + return 1 + print("strategy-source-ok") + return 0 + + +_COMMANDS: dict[str, Callable[[], int]] = { + "scored-create": _cmd_scored_create, + "scored-breakdown": _cmd_scored_breakdown, + "scored-equality": _cmd_scored_equality, + "scored-dedup": _cmd_scored_dedup, + "scored-from-relevance": _cmd_scored_from_relevance, + "scored-frozen": _cmd_scored_frozen, + "strategy-source": _cmd_strategy_source, +} + + +def main() -> int: + """Dispatch to the requested sub-command.""" + if len(sys.argv) < 2 or sys.argv[1] not in _COMMANDS: + valid = ", ".join(sorted(_COMMANDS)) + print(f"Usage: {sys.argv[0]} <{valid}>", file=sys.stderr) + return 1 + return _COMMANDS[sys.argv[1]]() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cleveragents/domain/contexts/__init__.py b/src/cleveragents/domain/contexts/__init__.py index d444aea88..2c9f19546 100644 --- a/src/cleveragents/domain/contexts/__init__.py +++ b/src/cleveragents/domain/contexts/__init__.py @@ -1,6 +1,11 @@ """Contexts domain module. -Contains domain models and business logic related to plan contexts. +Contains pipeline-specific domain models for the ACMS context assembly +pipeline, including ``ScoredFragment`` for scored/ranked fragments. """ -__all__ = [] +from cleveragents.domain.contexts.fragment import ScoredFragment + +__all__ = [ + "ScoredFragment", +] diff --git a/src/cleveragents/domain/contexts/fragment.py b/src/cleveragents/domain/contexts/fragment.py new file mode 100644 index 000000000..730f8cac8 --- /dev/null +++ b/src/cleveragents/domain/contexts/fragment.py @@ -0,0 +1,180 @@ +"""Pipeline-specific fragment models for the ACMS context assembly pipeline. + +Provides ``ScoredFragment``, a frozen Pydantic value object that wraps a +``ContextFragment`` with composite scoring information produced by the +``FragmentScorer`` pipeline stage. + +Equality and hashing are based on ``uko_node`` + ``detail_depth`` to support +deduplication in the pipeline without relying on object identity. + +Based on ``docs/specification.md`` ~line 42826 (ScoredFragment). +""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, field_validator, model_validator + +from cleveragents.domain.models.core.context_fragment import ContextFragment + + +class ScoredFragment(BaseModel, frozen=True): + """A ``ContextFragment`` annotated with a composite score. + + Produced by the ``FragmentScorer`` pipeline stage and consumed by + ``BudgetPacker`` for ranking during budget packing. The model is + frozen so that scored fragments are safe to share across pipeline + stages without defensive copies. + + Equality and hashing are based on ``fragment.uko_node`` and + ``fragment.detail_depth`` so that two ``ScoredFragment`` instances + wrapping the same logical content are considered duplicates + regardless of scoring differences. + + Based on ``docs/specification.md`` ~line 42826. + """ + + fragment: ContextFragment = Field( + ..., + description="The underlying context fragment being scored.", + ) + + composite_score: float = Field( + ..., + ge=0.0, + le=1.0, + description="Overall composite ranking score (0.0-1.0).", + ) + + score_breakdown: dict[str, float] = Field( + default_factory=dict, + description=( + "Component-level score breakdown. Keys are component names " + "(e.g. 'relevance', 'hierarchy', 'recency') and values are " + "individual scores in 0.0-1.0." + ), + ) + + rank: int = Field( + default=0, + ge=0, + description=( + "Ordinal rank assigned after scoring (0 = unranked / not yet " + "assigned). Lower values indicate higher priority." + ), + ) + + @field_validator("score_breakdown") + @classmethod + def _validate_score_breakdown(cls, v: dict[str, float]) -> dict[str, float]: + """Validate all component scores are in [0.0, 1.0] and deep-copy.""" + for name, score in v.items(): + if score < 0.0 or score > 1.0: + msg = ( + f"score_breakdown['{name}'] must be between 0.0 and 1.0, " + f"got {score}" + ) + raise ValueError(msg) + return dict(v) + + @model_validator(mode="after") + def _check_composite_within_components(self) -> ScoredFragment: + """Warn-free validation: composite_score is present and valid. + + The composite score does not need to equal the average of + components — strategies may apply custom weighting — but it + must be in range, which is already enforced by the field + constraint. + """ + return self + + # ------------------------------------------------------------------ + # Equality / hashing based on uko_node + detail_depth + # ------------------------------------------------------------------ + + def __eq__(self, other: object) -> bool: + """Two scored fragments are equal when their underlying fragments + share the same ``uko_node`` and ``detail_depth``.""" + if not isinstance(other, ScoredFragment): + return NotImplemented + return ( + self.fragment.uko_node == other.fragment.uko_node + and self.fragment.detail_depth == other.fragment.detail_depth + ) + + def __hash__(self) -> int: + """Hash based on ``uko_node`` + ``detail_depth`` for set/dict use.""" + return hash((self.fragment.uko_node, self.fragment.detail_depth)) + + # ------------------------------------------------------------------ + # Factory helpers + # ------------------------------------------------------------------ + + @staticmethod + def from_fragment( + fragment: ContextFragment, + composite_score: float, + score_breakdown: dict[str, float] | None = None, + rank: int = 0, + ) -> ScoredFragment: + """Create a ``ScoredFragment`` from an existing ``ContextFragment``. + + This is the preferred construction method for pipeline stages. + + Args: + fragment: The context fragment to wrap. + composite_score: The overall composite score (0.0-1.0). + score_breakdown: Optional component-level breakdown. + rank: Optional ordinal rank (0 = unranked). + + Returns: + A new frozen ``ScoredFragment`` instance. + """ + return ScoredFragment( + fragment=fragment, + composite_score=composite_score, + score_breakdown=score_breakdown if score_breakdown is not None else {}, + rank=rank, + ) + + @staticmethod + def from_relevance( + fragment: ContextFragment, + rank: int = 0, + ) -> ScoredFragment: + """Create a ``ScoredFragment`` using only the fragment's relevance_score. + + Convenience factory for simple scoring where the composite score + equals the fragment's built-in ``relevance_score``. + + Args: + fragment: The context fragment to wrap. + rank: Optional ordinal rank (0 = unranked). + + Returns: + A new frozen ``ScoredFragment`` with relevance as sole component. + """ + return ScoredFragment( + fragment=fragment, + composite_score=fragment.relevance_score, + score_breakdown={"relevance": fragment.relevance_score}, + rank=rank, + ) + + # ------------------------------------------------------------------ + # Convenience properties + # ------------------------------------------------------------------ + + @property + def uko_node(self) -> str: + """Shortcut to the underlying fragment's ``uko_node``.""" + return self.fragment.uko_node + + @property + def detail_depth(self) -> int: + """Shortcut to the underlying fragment's ``detail_depth``.""" + return self.fragment.detail_depth + + @property + def token_count(self) -> int: + """Shortcut to the underlying fragment's ``token_count``.""" + return self.fragment.token_count diff --git a/src/cleveragents/domain/models/core/context_fragment.py b/src/cleveragents/domain/models/core/context_fragment.py index 16634d0a0..b90f70146 100644 --- a/src/cleveragents/domain/models/core/context_fragment.py +++ b/src/cleveragents/domain/models/core/context_fragment.py @@ -102,6 +102,14 @@ class ContextFragment(BaseModel, frozen=True): description="Trace back to resource and location", ) + strategy_source: str = Field( + default="", + description=( + "Name of the strategy that produced this fragment. Empty string " + "means the strategy is unknown or not yet assigned." + ), + ) + tier: Literal["hot", "warm", "cold"] = Field(default="warm") metadata: dict[str, str] = Field(default_factory=dict)