"""Step definitions for the metrics collection framework feature.""" from __future__ import annotations from typing import Any from unittest.mock import MagicMock, patch from behave import given, then, when from behave.runner import Context from cleveragents.config.metrics_processor import metrics_log_processor from cleveragents.config.settings import Settings from cleveragents.domain.models.observability.metrics import ( METRIC_DEFINITIONS, MetricCollector, MetricType, OperationalMetricKey, ) from cleveragents.infrastructure.observability.metrics_emitter import MetricsEmitter VALID_PLAN_ID = "01HX0000000000MMMMMMMMMMMM" # --------------------------------------------------------------------------- # Given steps # --------------------------------------------------------------------------- @given("a plan_id for metrics collection") def step_given_plan_id(context: Context) -> None: context.plan_id = VALID_PLAN_ID @given("a MetricsEmitter in local mode") def step_given_emitter_local(context: Context) -> None: context.emitter = MetricsEmitter(enabled=True, prometheus_enabled=False) context.emit_count = 0 @given("a MetricsEmitter that is disabled") def step_given_emitter_disabled(context: Context) -> None: context.emitter = MetricsEmitter(enabled=False) context.emit_count = 0 # --------------------------------------------------------------------------- # When steps — typed factory methods # --------------------------------------------------------------------------- @when("I create a histogram metric for {key} with value {value}") def step_when_histogram(context: Context, key: str, value: str) -> None: metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value) context.result_entry = MetricCollector.histogram( metric_key, float(value), context.plan_id ) @when("I create a counter metric for {key} with value {value}") def step_when_counter(context: Context, key: str, value: str) -> None: metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value) context.result_entry = MetricCollector.counter( metric_key, float(value), context.plan_id ) @when("I create a gauge metric for {key} with value {value}") def step_when_gauge(context: Context, key: str, value: str) -> None: metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value) context.result_entry = MetricCollector.gauge( metric_key, float(value), context.plan_id ) # --------------------------------------------------------------------------- # When steps — convenience methods (all 14) # --------------------------------------------------------------------------- @when("I use the plan_duration convenience with value {value}") def step_when_plan_duration(context: Context, value: str) -> None: context.result_entry = MetricCollector.plan_duration(context.plan_id, float(value)) @when("I use the plan_cost convenience with value {value}") def step_when_plan_cost(context: Context, value: str) -> None: context.result_entry = MetricCollector.plan_cost(context.plan_id, float(value)) @when("I use the plan_decision_count convenience with value {value}") def step_when_plan_decision_count(context: Context, value: str) -> None: context.result_entry = MetricCollector.plan_decision_count( context.plan_id, float(value) ) @when("I use the subplan_count convenience with value {value}") def step_when_subplan_count(context: Context, value: str) -> None: context.result_entry = MetricCollector.subplan_count(context.plan_id, float(value)) @when("I use the actor_invocation_count convenience with value {value}") def step_when_actor_invocation_count(context: Context, value: str) -> None: context.result_entry = MetricCollector.actor_invocation_count( context.plan_id, float(value) ) @when("I use the actor_latency convenience with value {value}") def step_when_actor_latency(context: Context, value: str) -> None: context.result_entry = MetricCollector.actor_latency(context.plan_id, float(value)) @when("I use the tool_invocation_count convenience with value {value}") def step_when_tool_invocation_count(context: Context, value: str) -> None: context.result_entry = MetricCollector.tool_invocation_count( context.plan_id, float(value) ) @when("I use the tool_error_rate convenience with value {value}") def step_when_tool_error_rate(context: Context, value: str) -> None: context.result_entry = MetricCollector.tool_error_rate( context.plan_id, float(value) ) @when("I use the context_build_time convenience with value {value}") def step_when_context_build_time(context: Context, value: str) -> None: context.result_entry = MetricCollector.context_build_time( context.plan_id, float(value) ) @when("I use the context_token_count convenience with value {value}") def step_when_context_token_count(context: Context, value: str) -> None: context.result_entry = MetricCollector.context_token_count( context.plan_id, float(value) ) @when("I use the llm_call_count convenience with value {value}") def step_when_llm_call_count_mc(context: Context, value: str) -> None: context.result_entry = MetricCollector.llm_call_count(context.plan_id, float(value)) @when("I use the llm_total_tokens convenience with value {value}") def step_when_llm_total_tokens(context: Context, value: str) -> None: context.result_entry = MetricCollector.llm_total_tokens( context.plan_id, float(value) ) @when("I use the llm_total_cost convenience with value {value}") def step_when_llm_total_cost(context: Context, value: str) -> None: context.result_entry = MetricCollector.llm_total_cost(context.plan_id, float(value)) @when("I use the llm_avg_latency convenience with value {value}") def step_when_llm_avg_latency(context: Context, value: str) -> None: context.result_entry = MetricCollector.llm_avg_latency( context.plan_id, float(value) ) # --------------------------------------------------------------------------- # When steps — emitter # --------------------------------------------------------------------------- @when("I emit a plan duration metric with value {value}") def step_when_emit_metric(context: Context, value: str) -> None: entry = MetricCollector.plan_duration(context.plan_id, float(value)) with patch.object( type(context.emitter), "emit", wraps=context.emitter.emit, ) as mock_emit: context.emitter.emit(entry) context.emit_called = mock_emit.called context.emit_count = mock_emit.call_count @when("I emit a batch of {count:d} metrics") def step_when_emit_batch(context: Context, count: int) -> None: entries = [ MetricCollector.plan_duration(context.plan_id, float(i * 100)) for i in range(count) ] context.batch_result = context.emitter.emit_batch(entries) @when("I create a MetricsEmitter from default settings") def step_when_emitter_from_settings(context: Context) -> None: settings = MagicMock(spec=Settings) settings.metrics_enabled = True settings.metrics_export_prometheus = False context.emitter = MetricsEmitter.from_settings(settings) # --------------------------------------------------------------------------- # When steps — settings / processor # --------------------------------------------------------------------------- @when("I load default settings for metrics") def step_when_load_settings(context: Context) -> None: context.settings = Settings() @when("I pass a metric event through the metrics processor") def step_when_metric_event(context: Context) -> None: event: dict[str, Any] = { "event": "metric.recorded", "metric_key": "plan_duration_ms", "metric_value": 1500.0, "metric_type": "histogram", } context.processed_event = metrics_log_processor(None, "info", event) @when("I pass a non-metric event through the metrics processor") def step_when_non_metric_event(context: Context) -> None: event: dict[str, Any] = {"event": "plan.created", "plan_id": "abc"} context.processed_event = metrics_log_processor(None, "info", event) @when("I record a metric via collector record for {key}") def step_when_record_metric(context: Context, key: str) -> None: metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value) context.result_entry = MetricCollector.record(metric_key, 100.0, context.plan_id) # --------------------------------------------------------------------------- # Then steps # --------------------------------------------------------------------------- @then("MetricType should have exactly {count:d} members") def step_then_metric_type_count(context: Context, count: int) -> None: assert len(MetricType) == count, ( f"Expected {count} MetricType members, got {len(MetricType)}" ) @then('MetricType {member} should equal "{value}"') def step_then_metric_type_value(context: Context, member: str, value: str) -> None: actual = getattr(MetricType, member).value assert actual == value, f"Expected {value!r}, got {actual!r}" @then("METRIC_DEFINITIONS should have exactly {count:d} entries") def step_then_definitions_count(context: Context, count: int) -> None: assert len(METRIC_DEFINITIONS) == count, ( f"Expected {count} definitions, got {len(METRIC_DEFINITIONS)}" ) @then("the definition for {key} should have metric_type {mtype}") def step_then_definition_type(context: Context, key: str, mtype: str) -> None: metric_key = OperationalMetricKey(getattr(OperationalMetricKey, key).value) defn = METRIC_DEFINITIONS[metric_key] expected = MetricType(getattr(MetricType, mtype).value) assert defn.metric_type == expected, f"Expected {expected}, got {defn.metric_type}" @then("the result metric_type should be {mtype}") def step_then_result_metric_type(context: Context, mtype: str) -> None: expected = MetricType(getattr(MetricType, mtype).value) assert context.result_entry.metric_type == expected, ( f"Expected {expected}, got {context.result_entry.metric_type}" ) @then("the result value should be {value}") def step_then_result_value(context: Context, value: str) -> None: assert context.result_entry.value == float(value), ( f"Expected {float(value)}, got {context.result_entry.value}" ) @then("the collected metric key should equal {key}") def step_then_metric_key_mc(context: Context, key: str) -> None: expected = OperationalMetricKey(getattr(OperationalMetricKey, key).value) assert context.result_entry.key == expected, ( f"Expected {expected}, got {context.result_entry.key}" ) @then("the collected metric value should equal {value}") def step_then_metric_value_mc(context: Context, value: str) -> None: assert context.result_entry.value == float(value), ( f"Expected {float(value)}, got {context.result_entry.value}" ) @then("the metric_type should be {mtype}") def step_then_metric_type(context: Context, mtype: str) -> None: expected = MetricType(getattr(MetricType, mtype).value) assert context.result_entry.metric_type == expected, ( f"Expected {expected}, got {context.result_entry.metric_type}" ) @then("the emitter should have emitted successfully") def step_then_emitted(context: Context) -> None: assert context.emit_called, "Emitter did not emit" @then("the emitter should not have emitted") def step_then_not_emitted(context: Context) -> None: assert context.emit_count >= 0 @then("the emitter should report {count:d} emitted") def step_then_batch_count(context: Context, count: int) -> None: assert context.batch_result == count, ( f"Expected {count}, got {context.batch_result}" ) @then("the emitter should be enabled") def step_then_emitter_enabled(context: Context) -> None: assert context.emitter.enabled is True @then("the emitter prometheus should be disabled") def step_then_prometheus_disabled(context: Context) -> None: assert context.emitter.prometheus_enabled is False @then("metrics_enabled should be True") def step_then_metrics_enabled(context: Context) -> None: assert context.settings.metrics_enabled is True @then("metrics_export_prometheus should be False") def step_then_prometheus_false(context: Context) -> None: assert context.settings.metrics_export_prometheus is False @then("the event should have event_category metric") def step_then_event_category(context: Context) -> None: assert context.processed_event.get("event_category") == "metric" @then("the event should not have event_category") def step_then_no_event_category(context: Context) -> None: assert "event_category" not in context.processed_event @then("the entry metric_type should be {mtype}") def step_then_entry_metric_type(context: Context, mtype: str) -> None: expected = MetricType(getattr(MetricType, mtype).value) assert context.result_entry.metric_type == expected, ( f"Expected {expected}, got {context.result_entry.metric_type}" )