cleveragents-core/features/steps/metrics_collection_steps.py

"""Step definitions for the metrics collection framework feature."""

from __future__ import annotations

from typing import Any
from unittest.mock import MagicMock, patch

from behave import given, then, when
from behave.runner import Context

from cleveragents.config.metrics_processor import metrics_log_processor
from cleveragents.config.settings import Settings
from cleveragents.domain.models.observability.metrics import (
    METRIC_DEFINITIONS,
    MetricCollector,
    MetricType,
    OperationalMetricKey,
)
from cleveragents.infrastructure.observability.metrics_emitter import MetricsEmitter

VALID_PLAN_ID = "01HX0000000000MMMMMMMMMMMM"


# ---------------------------------------------------------------------------
# Given steps
# ---------------------------------------------------------------------------


@given("a plan_id for metrics collection")
def step_given_plan_id(context: Context) -> None:
    context.plan_id = VALID_PLAN_ID


@given("a MetricsEmitter in local mode")
def step_given_emitter_local(context: Context) -> None:
    context.emitter = MetricsEmitter(enabled=True, prometheus_enabled=False)
    context.emit_count = 0


@given("a MetricsEmitter that is disabled")
def step_given_emitter_disabled(context: Context) -> None:
    context.emitter = MetricsEmitter(enabled=False)
    context.emit_count = 0


# ---------------------------------------------------------------------------
# When steps — typed factory methods
# ---------------------------------------------------------------------------


@when("I create a histogram metric for {key} with value {value}")
def step_when_histogram(context: Context, key: str, value: str) -> None:
    metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    context.result_entry = MetricCollector.histogram(
        metric_key, float(value), context.plan_id
    )


@when("I create a counter metric for {key} with value {value}")
def step_when_counter(context: Context, key: str, value: str) -> None:
    metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    context.result_entry = MetricCollector.counter(
        metric_key, float(value), context.plan_id
    )


@when("I create a gauge metric for {key} with value {value}")
def step_when_gauge(context: Context, key: str, value: str) -> None:
    metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    context.result_entry = MetricCollector.gauge(
        metric_key, float(value), context.plan_id
    )


# ---------------------------------------------------------------------------
# When steps — convenience methods (all 14)
# ---------------------------------------------------------------------------


@when("I use the plan_duration convenience with value {value}")
def step_when_plan_duration(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.plan_duration(context.plan_id, float(value))


@when("I use the plan_cost convenience with value {value}")
def step_when_plan_cost(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.plan_cost(context.plan_id, float(value))


@when("I use the plan_decision_count convenience with value {value}")
def step_when_plan_decision_count(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.plan_decision_count(
        context.plan_id, float(value)
    )


@when("I use the subplan_count convenience with value {value}")
def step_when_subplan_count(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.subplan_count(context.plan_id, float(value))


@when("I use the actor_invocation_count convenience with value {value}")
def step_when_actor_invocation_count(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.actor_invocation_count(
        context.plan_id, float(value)
    )


@when("I use the actor_latency convenience with value {value}")
def step_when_actor_latency(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.actor_latency(context.plan_id, float(value))


@when("I use the tool_invocation_count convenience with value {value}")
def step_when_tool_invocation_count(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.tool_invocation_count(
        context.plan_id, float(value)
    )


@when("I use the tool_error_rate convenience with value {value}")
def step_when_tool_error_rate(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.tool_error_rate(
        context.plan_id, float(value)
    )


@when("I use the context_build_time convenience with value {value}")
def step_when_context_build_time(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.context_build_time(
        context.plan_id, float(value)
    )


@when("I use the context_token_count convenience with value {value}")
def step_when_context_token_count(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.context_token_count(
        context.plan_id, float(value)
    )


@when("I use the llm_call_count convenience with value {value}")
def step_when_llm_call_count_mc(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.llm_call_count(context.plan_id, float(value))


@when("I use the llm_total_tokens convenience with value {value}")
def step_when_llm_total_tokens(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.llm_total_tokens(
        context.plan_id, float(value)
    )


@when("I use the llm_total_cost convenience with value {value}")
def step_when_llm_total_cost(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.llm_total_cost(context.plan_id, float(value))


@when("I use the llm_avg_latency convenience with value {value}")
def step_when_llm_avg_latency(context: Context, value: str) -> None:
    context.result_entry = MetricCollector.llm_avg_latency(
        context.plan_id, float(value)
    )


# ---------------------------------------------------------------------------
# When steps — emitter
# ---------------------------------------------------------------------------


@when("I emit a plan duration metric with value {value}")
def step_when_emit_metric(context: Context, value: str) -> None:
    entry = MetricCollector.plan_duration(context.plan_id, float(value))
    with patch.object(
        type(context.emitter),
        "emit",
        wraps=context.emitter.emit,
    ) as mock_emit:
        context.emitter.emit(entry)
        context.emit_called = mock_emit.called
        context.emit_count = mock_emit.call_count


@when("I emit a batch of {count:d} metrics")
def step_when_emit_batch(context: Context, count: int) -> None:
    entries = [
        MetricCollector.plan_duration(context.plan_id, float(i * 100))
        for i in range(count)
    ]
    context.batch_result = context.emitter.emit_batch(entries)


@when("I create a MetricsEmitter from default settings")
def step_when_emitter_from_settings(context: Context) -> None:
    settings = MagicMock(spec=Settings)
    settings.metrics_enabled = True
    settings.metrics_export_prometheus = False
    context.emitter = MetricsEmitter.from_settings(settings)


# ---------------------------------------------------------------------------
# When steps — settings / processor
# ---------------------------------------------------------------------------


@when("I load default settings for metrics")
def step_when_load_settings(context: Context) -> None:
    context.settings = Settings()


@when("I pass a metric event through the metrics processor")
def step_when_metric_event(context: Context) -> None:
    event: dict[str, Any] = {
        "event": "metric.recorded",
        "metric_key": "plan_duration_ms",
        "metric_value": 1500.0,
        "metric_type": "histogram",
    }
    context.processed_event = metrics_log_processor(None, "info", event)


@when("I pass a non-metric event through the metrics processor")
def step_when_non_metric_event(context: Context) -> None:
    event: dict[str, Any] = {"event": "plan.created", "plan_id": "abc"}
    context.processed_event = metrics_log_processor(None, "info", event)


@when("I record a metric via collector record for {key}")
def step_when_record_metric(context: Context, key: str) -> None:
    metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    context.result_entry = MetricCollector.record(metric_key, 100.0, context.plan_id)


# ---------------------------------------------------------------------------
# Then steps
# ---------------------------------------------------------------------------


@then("MetricType should have exactly {count:d} members")
def step_then_metric_type_count(context: Context, count: int) -> None:
    assert len(MetricType) == count, (
        f"Expected {count} MetricType members, got {len(MetricType)}"
    )


@then('MetricType {member} should equal "{value}"')
def step_then_metric_type_value(context: Context, member: str, value: str) -> None:
    actual = getattr(MetricType, member).value
    assert actual == value, f"Expected {value!r}, got {actual!r}"


@then("METRIC_DEFINITIONS should have exactly {count:d} entries")
def step_then_definitions_count(context: Context, count: int) -> None:
    assert len(METRIC_DEFINITIONS) == count, (
        f"Expected {count} definitions, got {len(METRIC_DEFINITIONS)}"
    )


@then("the definition for {key} should have metric_type {mtype}")
def step_then_definition_type(context: Context, key: str, mtype: str) -> None:
    metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    defn = METRIC_DEFINITIONS[metric_key]
    expected = MetricType(getattr(MetricType, mtype).value)
    assert defn.metric_type == expected, f"Expected {expected}, got {defn.metric_type}"


@then("the result metric_type should be {mtype}")
def step_then_result_metric_type(context: Context, mtype: str) -> None:
    expected = MetricType(getattr(MetricType, mtype).value)
    assert context.result_entry.metric_type == expected, (
        f"Expected {expected}, got {context.result_entry.metric_type}"
    )


@then("the result value should be {value}")
def step_then_result_value(context: Context, value: str) -> None:
    assert context.result_entry.value == float(value), (
        f"Expected {float(value)}, got {context.result_entry.value}"
    )


@then("the collected metric key should equal {key}")
def step_then_metric_key_mc(context: Context, key: str) -> None:
    expected = OperationalMetricKey(getattr(OperationalMetricKey, key).value)
    assert context.result_entry.key == expected, (
        f"Expected {expected}, got {context.result_entry.key}"
    )


@then("the collected metric value should equal {value}")
def step_then_metric_value_mc(context: Context, value: str) -> None:
    assert context.result_entry.value == float(value), (
        f"Expected {float(value)}, got {context.result_entry.value}"
    )


@then("the metric_type should be {mtype}")
def step_then_metric_type(context: Context, mtype: str) -> None:
    expected = MetricType(getattr(MetricType, mtype).value)
    assert context.result_entry.metric_type == expected, (
        f"Expected {expected}, got {context.result_entry.metric_type}"
    )


@then("the emitter should have emitted successfully")
def step_then_emitted(context: Context) -> None:
    assert context.emit_called, "Emitter did not emit"


@then("the emitter should not have emitted")
def step_then_not_emitted(context: Context) -> None:
    assert context.emit_count >= 0


@then("the emitter should report {count:d} emitted")
def step_then_batch_count(context: Context, count: int) -> None:
    assert context.batch_result == count, (
        f"Expected {count}, got {context.batch_result}"
    )


@then("the emitter should be enabled")
def step_then_emitter_enabled(context: Context) -> None:
    assert context.emitter.enabled is True


@then("the emitter prometheus should be disabled")
def step_then_prometheus_disabled(context: Context) -> None:
    assert context.emitter.prometheus_enabled is False


@then("metrics_enabled should be True")
def step_then_metrics_enabled(context: Context) -> None:
    assert context.settings.metrics_enabled is True


@then("metrics_export_prometheus should be False")
def step_then_prometheus_false(context: Context) -> None:
    assert context.settings.metrics_export_prometheus is False


@then("the event should have event_category metric")
def step_then_event_category(context: Context) -> None:
    assert context.processed_event.get("event_category") == "metric"


@then("the event should not have event_category")
def step_then_no_event_category(context: Context) -> None:
    assert "event_category" not in context.processed_event


@then("the entry metric_type should be {mtype}")
def step_then_entry_metric_type(context: Context, mtype: str) -> None:
    expected = MetricType(getattr(MetricType, mtype).value)
    assert context.result_entry.metric_type == expected, (
        f"Expected {expected}, got {context.result_entry.metric_type}"
    )